wreq-rb 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +257 -102
  3. data/ext/wreq_rb/Cargo.toml +7 -4
  4. data/ext/wreq_rb/src/client.rs +105 -15
  5. data/ext/wreq_rb/src/response.rs +28 -3
  6. data/lib/wreq-rb/version.rb +1 -1
  7. data/patches/0001-add-transfer-size-tracking.patch +11 -15
  8. data/vendor/wreq/Cargo.toml +9 -8
  9. data/vendor/wreq/README.md +5 -5
  10. data/vendor/wreq/bench/support/bench.rs +6 -2
  11. data/vendor/wreq/bench/support/client.rs +88 -1
  12. data/vendor/wreq/bench/support/exec.rs +0 -0
  13. data/vendor/wreq/bench/support/rt.rs +34 -0
  14. data/vendor/wreq/bench/support/server.rs +1 -1
  15. data/vendor/wreq/bench/support.rs +1 -15
  16. data/vendor/wreq/examples/cert_store.rs +13 -13
  17. data/vendor/wreq/examples/request_with_emulate.rs +1 -1
  18. data/vendor/wreq/examples/tcp_linger.rs +22 -0
  19. data/vendor/wreq/src/client/layer/client/pool.rs +17 -17
  20. data/vendor/wreq/src/client/layer/client.rs +2 -0
  21. data/vendor/wreq/src/client/layer/decoder.rs +71 -17
  22. data/vendor/wreq/src/client/layer/redirect/future.rs +49 -63
  23. data/vendor/wreq/src/client/layer/redirect/policy.rs +2 -26
  24. data/vendor/wreq/src/client/layer/redirect.rs +48 -60
  25. data/vendor/wreq/src/client/layer/retry.rs +12 -15
  26. data/vendor/wreq/src/client/layer/timeout/body.rs +27 -21
  27. data/vendor/wreq/src/client/layer/timeout/future.rs +33 -58
  28. data/vendor/wreq/src/client/layer/timeout.rs +8 -14
  29. data/vendor/wreq/src/client/request.rs +4 -0
  30. data/vendor/wreq/src/client.rs +53 -31
  31. data/vendor/wreq/src/conn/connector.rs +99 -129
  32. data/vendor/wreq/src/conn/http.rs +25 -18
  33. data/vendor/wreq/src/conn/net/tcp.rs +601 -107
  34. data/vendor/wreq/src/conn/proxy/socks.rs +6 -6
  35. data/vendor/wreq/src/conn/timeout.rs +166 -0
  36. data/vendor/wreq/src/conn.rs +5 -4
  37. data/vendor/wreq/src/cookie/jar.rs +1225 -0
  38. data/vendor/wreq/src/cookie/store.rs +321 -0
  39. data/vendor/wreq/src/cookie.rs +108 -612
  40. data/vendor/wreq/src/dns/resolve.rs +8 -2
  41. data/vendor/wreq/src/dns.rs +4 -4
  42. data/vendor/wreq/src/error.rs +53 -20
  43. data/vendor/wreq/src/lib.rs +1 -0
  44. data/vendor/wreq/src/proxy/matcher.rs +26 -12
  45. data/vendor/wreq/src/proxy/win.rs +39 -9
  46. data/vendor/wreq/src/redirect.rs +515 -100
  47. data/vendor/wreq/src/tls/conn.rs +3 -11
  48. data/vendor/wreq/src/tls/session.rs +7 -8
  49. data/vendor/wreq/src/tls/trust/store.rs +4 -4
  50. data/vendor/wreq/src/util.rs +23 -0
  51. data/vendor/wreq/tests/badssl.rs +72 -7
  52. data/vendor/wreq/tests/brotli.rs +1 -1
  53. data/vendor/wreq/tests/client.rs +24 -0
  54. data/vendor/wreq/tests/connector_layers.rs +8 -4
  55. data/vendor/wreq/tests/cookie.rs +59 -0
  56. data/vendor/wreq/tests/deflate.rs +1 -1
  57. data/vendor/wreq/tests/gzip.rs +53 -1
  58. data/vendor/wreq/tests/layers.rs +8 -4
  59. data/vendor/wreq/tests/redirect.rs +180 -97
  60. data/vendor/wreq/tests/timeouts.rs +47 -12
  61. data/vendor/wreq/tests/zstd.rs +1 -1
  62. metadata +8 -2
@@ -222,17 +222,9 @@ impl TlsConnector {
222
222
  /// it (otherwise, boring will fail to parse the host as an IP address, eventually
223
223
  /// causing the handshake to fail due a hostname verification error).
224
224
  fn normalize_host(host: &str) -> &str {
225
- if host.is_empty() {
226
- return host;
227
- }
228
-
229
- let last = host.len() - 1;
230
- let mut chars = host.chars();
231
-
232
- if let (Some('['), Some(']')) = (chars.next(), chars.last()) {
233
- if host[1..last].parse::<std::net::Ipv6Addr>().is_ok() {
234
- return &host[1..last];
235
- }
225
+ let normalized = crate::util::strip_ipv6_brackets(host);
226
+ if normalized.len() != host.len() && normalized.parse::<std::net::Ipv6Addr>().is_ok() {
227
+ return normalized;
236
228
  }
237
229
 
238
230
  host
@@ -189,14 +189,13 @@ impl TlsSessionCache for LruTlsSessionCache {
189
189
  // https://tools.ietf.org/html/rfc8446#appendix-C.4
190
190
  // OpenSSL will remove the session from its cache after the handshake completes anyway, but
191
191
  // this ensures that concurrent handshakes don't end up with the same session.
192
- if session.protocol_version() == TlsVersion::TLS_1_3 {
193
- if let Some(key) = inner.reverse.remove(&session) {
194
- if let Entry::Occupied(mut entry) = inner.per_host_sessions.entry(key) {
195
- entry.get_mut().pop(&session);
196
- if entry.get().is_empty() {
197
- entry.remove();
198
- }
199
- }
192
+ if session.protocol_version() == TlsVersion::TLS_1_3
193
+ && let Some(key) = inner.reverse.remove(&session)
194
+ && let Entry::Occupied(mut entry) = inner.per_host_sessions.entry(key)
195
+ {
196
+ entry.get_mut().pop(&session);
197
+ if entry.get().is_empty() {
198
+ entry.remove();
200
199
  }
201
200
  }
202
201
 
@@ -106,10 +106,10 @@ impl CertStoreBuilder {
106
106
  /// environment variables if present, or defaults specified at OpenSSL
107
107
  /// build time otherwise.
108
108
  pub fn set_default_paths(mut self) -> Self {
109
- if let Ok(ref mut builder) = self.builder {
110
- if let Err(err) = builder.set_default_paths() {
111
- self.builder = Err(Error::tls(err));
112
- }
109
+ if let Ok(ref mut builder) = self.builder
110
+ && let Err(err) = builder.set_default_paths()
111
+ {
112
+ self.builder = Err(Error::tls(err));
113
113
  }
114
114
  self
115
115
  }
@@ -4,6 +4,14 @@ use bytes::Bytes;
4
4
 
5
5
  use crate::header::{Entry, HeaderMap, HeaderValue, OccupiedEntry};
6
6
 
7
+ /// Removes the square brackets around an IPv6 host literal as required by
8
+ /// [RFC 3986, Section 3.2.2](https://www.rfc-editor.org/rfc/rfc3986.html#section-3.2.2).
9
+ pub(crate) fn strip_ipv6_brackets(host: &str) -> &str {
10
+ host.strip_prefix('[')
11
+ .and_then(|host| host.strip_suffix(']'))
12
+ .unwrap_or(host)
13
+ }
14
+
7
15
  pub(crate) fn basic_auth<U, P>(username: U, password: Option<P>) -> HeaderValue
8
16
  where
9
17
  U: fmt::Display,
@@ -120,3 +128,18 @@ impl fmt::Display for Escape<'_> {
120
128
  Ok(())
121
129
  }
122
130
  }
131
+
132
+ #[cfg(test)]
133
+ mod tests {
134
+ use super::strip_ipv6_brackets;
135
+
136
+ #[test]
137
+ fn strips_ipv6_brackets() {
138
+ assert_eq!(strip_ipv6_brackets("[::1]"), "::1");
139
+ assert_eq!(strip_ipv6_brackets("::1"), "::1");
140
+ assert_eq!(strip_ipv6_brackets("example.com"), "example.com");
141
+ assert_eq!(strip_ipv6_brackets("[example.com"), "[example.com");
142
+ assert_eq!(strip_ipv6_brackets("example.com]"), "example.com]");
143
+ assert_eq!(strip_ipv6_brackets("[[::1]]"), "[::1]");
144
+ }
145
+ }
@@ -1,7 +1,7 @@
1
- use std::time::Duration;
1
+ use std::{error::Error as StdError, io, time::Duration};
2
2
 
3
3
  use wreq::{
4
- Client,
4
+ Client, retry,
5
5
  tls::{AlpsProtocol, TlsInfo, TlsOptions, TlsVersion, trust::CertStore},
6
6
  };
7
7
 
@@ -11,11 +11,40 @@ macro_rules! join {
11
11
  };
12
12
  }
13
13
 
14
+ fn badssl_connection_reset_retry_policy() -> retry::Policy {
15
+ retry::Policy::default()
16
+ .max_retries_per_request(10)
17
+ .no_budget()
18
+ .classify_fn(|req_rep| {
19
+ if req_rep.error().is_some_and(is_connection_reset) {
20
+ req_rep.retryable()
21
+ } else {
22
+ req_rep.success()
23
+ }
24
+ })
25
+ }
26
+
27
+ fn is_connection_reset(err: &(dyn StdError + 'static)) -> bool {
28
+ let mut source = Some(err);
29
+
30
+ while let Some(err) = source {
31
+ if let Some(io) = err.downcast_ref::<io::Error>()
32
+ && io.kind() == io::ErrorKind::ConnectionReset
33
+ {
34
+ return true;
35
+ }
36
+
37
+ source = err.source();
38
+ }
39
+
40
+ false
41
+ }
42
+
14
43
  #[tokio::test]
15
44
  async fn test_badssl_modern() {
16
45
  let text = Client::builder()
17
46
  .no_proxy()
18
- .connect_timeout(Duration::from_secs(360))
47
+ .retry(badssl_connection_reset_retry_policy())
19
48
  .build()
20
49
  .unwrap()
21
50
  .get("https://mozilla-modern.badssl.com/")
@@ -26,15 +55,15 @@ async fn test_badssl_modern() {
26
55
  .await
27
56
  .unwrap();
28
57
 
29
- assert!(!text.is_empty());
58
+ assert!(text.contains("<title>mozilla-modern.badssl.com</title>"));
30
59
  }
31
60
 
32
61
  #[tokio::test]
33
62
  async fn test_badssl_self_signed() {
34
63
  let text = Client::builder()
35
64
  .tls_cert_verification(false)
36
- .connect_timeout(Duration::from_secs(360))
37
65
  .no_proxy()
66
+ .retry(badssl_connection_reset_retry_policy())
38
67
  .build()
39
68
  .unwrap()
40
69
  .get("https://self-signed.badssl.com/")
@@ -45,8 +74,39 @@ async fn test_badssl_self_signed() {
45
74
  .await
46
75
  .unwrap();
47
76
 
48
- assert!(!text.is_empty());
77
+ assert!(text.contains("<title>self-signed.badssl.com</title>"));
78
+ }
79
+
80
+ #[tokio::test]
81
+ async fn test_badssl_wrong_host() {
82
+ let text = Client::builder()
83
+ .tls_verify_hostname(false)
84
+ .no_proxy()
85
+ .retry(badssl_connection_reset_retry_policy())
86
+ .build()
87
+ .unwrap()
88
+ .get("https://wrong.host.badssl.com/")
89
+ .send()
90
+ .await
91
+ .unwrap()
92
+ .text()
93
+ .await
94
+ .unwrap();
95
+
96
+ assert!(text.contains("<title>wrong.host.badssl.com</title>"));
97
+
98
+ let result = Client::builder()
99
+ .tls_verify_hostname(false)
100
+ .no_proxy()
101
+ .build()
102
+ .unwrap()
103
+ .get("https://self-signed.badssl.com/")
104
+ .send()
105
+ .await;
106
+
107
+ assert!(result.is_err());
49
108
  }
109
+
50
110
  const CURVES_LIST: &str = join!(
51
111
  ":",
52
112
  "X25519",
@@ -72,6 +132,7 @@ async fn test_3des_support() -> wreq::Result<()> {
72
132
  let client = Client::builder()
73
133
  .tls_options(tls_options)
74
134
  .tls_cert_verification(false)
135
+ .retry(badssl_connection_reset_retry_policy())
75
136
  .connect_timeout(Duration::from_secs(360))
76
137
  .build()?;
77
138
 
@@ -105,6 +166,7 @@ async fn test_firefox_7x_100_cipher() -> wreq::Result<()> {
105
166
  let client = Client::builder()
106
167
  .tls_options(tls_options)
107
168
  .tls_cert_verification(false)
169
+ .retry(badssl_connection_reset_retry_policy())
108
170
  .connect_timeout(Duration::from_secs(360))
109
171
  .build()?;
110
172
 
@@ -189,8 +251,9 @@ async fn test_aes_hw_override() -> wreq::Result<()> {
189
251
  async fn test_tls_self_signed_cert() {
190
252
  let client = Client::builder()
191
253
  .tls_cert_verification(false)
192
- .connect_timeout(Duration::from_secs(360))
193
254
  .tls_info(true)
255
+ .retry(badssl_connection_reset_retry_policy())
256
+ .no_proxy()
194
257
  .build()
195
258
  .unwrap();
196
259
 
@@ -213,6 +276,8 @@ async fn test_tls_self_signed_cert() {
213
276
 
214
277
  let client = Client::builder()
215
278
  .tls_cert_store(self_signed_cert_store)
279
+ .retry(badssl_connection_reset_retry_policy())
280
+ .no_proxy()
216
281
  .build()
217
282
  .unwrap();
218
283
 
@@ -101,7 +101,7 @@ async fn brotli_case(response_size: usize, chunk_size: usize) {
101
101
  Content-Encoding: br\r\n\
102
102
  Content-Length: {}\r\n\
103
103
  \r\n",
104
- &brotlied_content.len()
104
+ brotlied_content.len()
105
105
  )
106
106
  .into_bytes();
107
107
  response.extend(&brotlied_content);
@@ -1090,3 +1090,27 @@ async fn response_trailers() {
1090
1090
  assert_eq!(trailers["chunky-trailer1"], "value1");
1091
1091
  assert_eq!(trailers["chunky-trailer2"], "value2");
1092
1092
  }
1093
+
1094
+ #[tokio::test]
1095
+ async fn dns_resolution_failure_is_dns_error() {
1096
+ let _ = env_logger::builder().is_test(true).try_init();
1097
+
1098
+ struct FailingResolver;
1099
+
1100
+ impl wreq::dns::Resolve for FailingResolver {
1101
+ fn resolve(&self, _name: wreq::dns::Name) -> reqwest::dns::Resolving {
1102
+ Box::pin(async { Err("simulated resolver failure".into()) })
1103
+ }
1104
+ }
1105
+
1106
+ let client = Client::builder()
1107
+ .no_proxy()
1108
+ .dns_resolver(FailingResolver)
1109
+ .build()
1110
+ .expect("client builder");
1111
+
1112
+ let err = client.get("http://hyper.rs").send().await.unwrap_err();
1113
+
1114
+ assert!(err.is_dns(), "expected a DNS error, got: {err:?}");
1115
+ assert!(err.is_connect(), "expected is_connect() to also be true");
1116
+ }
@@ -31,15 +31,17 @@ async fn non_op_layer() {
31
31
  async fn non_op_layer_with_timeout() {
32
32
  let _ = env_logger::try_init();
33
33
 
34
+ let server = server::http(move |_req| async { http::Response::default() });
35
+
34
36
  let client = Client::builder()
35
37
  .connector_layer(Identity::new())
38
+ .connector_layer(DelayLayer::new(Duration::from_secs(60)))
36
39
  .connect_timeout(Duration::from_millis(200))
37
40
  .no_proxy()
38
41
  .build()
39
42
  .unwrap();
40
43
 
41
- // never returns
42
- let url = "http://192.0.2.1:81/slow";
44
+ let url = format!("http://{}", server.addr());
43
45
 
44
46
  let res = client.get(url).send().await;
45
47
 
@@ -52,14 +54,16 @@ async fn non_op_layer_with_timeout() {
52
54
  async fn with_connect_timeout_layer_never_returning() {
53
55
  let _ = env_logger::try_init();
54
56
 
57
+ let server = server::http(move |_req| async { http::Response::default() });
58
+
55
59
  let client = Client::builder()
60
+ .connector_layer(DelayLayer::new(Duration::from_secs(60)))
56
61
  .connector_layer(TimeoutLayer::new(Duration::from_millis(100)))
57
62
  .no_proxy()
58
63
  .build()
59
64
  .unwrap();
60
65
 
61
- // never returns
62
- let url = "http://192.0.2.1:81/slow";
66
+ let url = format!("http://{}", server.addr());
63
67
 
64
68
  let res = client.get(url).send().await;
65
69
 
@@ -1,5 +1,7 @@
1
1
  mod support;
2
2
 
3
+ use std::sync::Arc;
4
+
3
5
  use http::{Version, header::COOKIE};
4
6
  use support::server;
5
7
  use wreq::{Client, cookie::Jar};
@@ -302,3 +304,60 @@ async fn cookie_request_level_compression() {
302
304
  .await
303
305
  .unwrap();
304
306
  }
307
+
308
+ #[tokio::test]
309
+ async fn cookie_request_order_matches_chromium() {
310
+ let server = server::http(|req| async move {
311
+ match req.uri().path() {
312
+ "/foo/bar/http1" => {
313
+ assert_eq!(req.version(), Version::HTTP_11);
314
+ assert_eq!(
315
+ req.headers()
316
+ .get(COOKIE)
317
+ .and_then(|value| value.to_str().ok()),
318
+ Some("B=B3; A=A3; B=B2; A=A2; B=B1; A=A1")
319
+ );
320
+ }
321
+ "/foo/bar/http2" => {
322
+ assert_eq!(req.version(), Version::HTTP_2);
323
+ let cookies = req
324
+ .headers()
325
+ .get_all(COOKIE)
326
+ .iter()
327
+ .map(|value| value.to_str().unwrap())
328
+ .collect::<Vec<_>>();
329
+ assert_eq!(cookies, ["B=B3", "A=A3", "B=B2", "A=A2", "B=B1", "A=A1"]);
330
+ }
331
+ path => panic!("unexpected request path: {path}"),
332
+ }
333
+
334
+ http::Response::default()
335
+ });
336
+
337
+ let base_url = format!("http://{}", server.addr());
338
+ let jar = Arc::new(Jar::default());
339
+ for cookie in [
340
+ "B=B1; Path=/",
341
+ "B=B2; Path=/foo",
342
+ "B=B3; Path=/foo/bar",
343
+ "A=A1; Path=/",
344
+ "A=A2; Path=/foo",
345
+ "A=A3; Path=/foo/bar",
346
+ ] {
347
+ jar.add(cookie, &base_url);
348
+ }
349
+
350
+ let client = Client::builder().cookie_provider(jar).build().unwrap();
351
+ client
352
+ .get(format!("{base_url}/foo/bar/http1"))
353
+ .version(Version::HTTP_11)
354
+ .send()
355
+ .await
356
+ .unwrap();
357
+ client
358
+ .get(format!("{base_url}/foo/bar/http2"))
359
+ .version(Version::HTTP_2)
360
+ .send()
361
+ .await
362
+ .unwrap();
363
+ }
@@ -100,7 +100,7 @@ async fn deflate_case(response_size: usize, chunk_size: usize) {
100
100
  Content-Encoding: deflate\r\n\
101
101
  Content-Length: {}\r\n\
102
102
  \r\n",
103
- &deflated_content.len()
103
+ deflated_content.len()
104
104
  )
105
105
  .into_bytes();
106
106
  response.extend(&deflated_content);
@@ -4,6 +4,7 @@ use std::io::Write;
4
4
  use flate2::{Compression, write::GzEncoder};
5
5
  use support::server;
6
6
  use tokio::io::AsyncWriteExt;
7
+ use wreq::header;
7
8
 
8
9
  #[tokio::test]
9
10
  async fn gzip_response() {
@@ -82,6 +83,57 @@ async fn test_accept_encoding_header_is_not_changed_if_set() {
82
83
  assert_eq!(res.status(), wreq::StatusCode::OK);
83
84
  }
84
85
 
86
+ #[tokio::test]
87
+ async fn test_range_requests_use_identity_encoding() {
88
+ let server = server::http(move |req| async move {
89
+ match req.uri().path() {
90
+ "/range" => {
91
+ assert_eq!(req.headers()[header::RANGE], "bytes=0-3");
92
+ assert_eq!(req.headers()[header::ACCEPT_ENCODING], "identity");
93
+ }
94
+ "/regular" => {
95
+ assert!(
96
+ req.headers()[header::ACCEPT_ENCODING]
97
+ .to_str()
98
+ .unwrap()
99
+ .contains("gzip")
100
+ );
101
+ }
102
+ "/disabled" => {
103
+ assert_eq!(req.headers()[header::ACCEPT_ENCODING], "gzip");
104
+ }
105
+ path => panic!("unexpected path: {path}"),
106
+ }
107
+
108
+ http::Response::default()
109
+ });
110
+ let base = format!("http://{}", server.addr());
111
+
112
+ let client = wreq::Client::new();
113
+ client
114
+ .get(format!("{base}/range"))
115
+ .header(header::RANGE, "bytes=0-3")
116
+ .header(header::ACCEPT_ENCODING, "gzip")
117
+ .send()
118
+ .await
119
+ .unwrap();
120
+ client.get(format!("{base}/regular")).send().await.unwrap();
121
+
122
+ wreq::Client::builder()
123
+ .no_gzip()
124
+ .no_brotli()
125
+ .no_zstd()
126
+ .no_deflate()
127
+ .build()
128
+ .unwrap()
129
+ .get(format!("{base}/disabled"))
130
+ .header(header::RANGE, "bytes=0-3")
131
+ .header(header::ACCEPT_ENCODING, "gzip")
132
+ .send()
133
+ .await
134
+ .unwrap();
135
+ }
136
+
85
137
  async fn gzip_case(response_size: usize, chunk_size: usize) {
86
138
  use futures_util::stream::StreamExt;
87
139
 
@@ -100,7 +152,7 @@ async fn gzip_case(response_size: usize, chunk_size: usize) {
100
152
  Content-Encoding: gzip\r\n\
101
153
  Content-Length: {}\r\n\
102
154
  \r\n",
103
- &gzipped_content.len()
155
+ gzipped_content.len()
104
156
  )
105
157
  .into_bytes();
106
158
  response.extend(&gzipped_content);
@@ -34,15 +34,17 @@ async fn non_op_layer() {
34
34
  async fn non_op_layer_with_timeout() {
35
35
  let _ = env_logger::try_init();
36
36
 
37
+ let server = server::http(move |_req| async { http::Response::default() });
38
+
37
39
  let client = Client::builder()
38
40
  .layer(Identity::new())
41
+ .connector_layer(DelayLayer::new(Duration::from_secs(60)))
39
42
  .connect_timeout(Duration::from_millis(200))
40
43
  .no_proxy()
41
44
  .build()
42
45
  .unwrap();
43
46
 
44
- // never returns
45
- let url = "http://192.0.2.1:81/slow";
47
+ let url = format!("http://{}", server.addr());
46
48
 
47
49
  let res = client.get(url).send().await;
48
50
 
@@ -55,14 +57,16 @@ async fn non_op_layer_with_timeout() {
55
57
  async fn with_connect_timeout_layer_never_returning() {
56
58
  let _ = env_logger::try_init();
57
59
 
60
+ let server = server::http(move |_req| async { http::Response::default() });
61
+
58
62
  let client = Client::builder()
63
+ .connector_layer(DelayLayer::new(Duration::from_secs(60)))
59
64
  .layer(TimeoutLayer::new(Duration::from_millis(100)))
60
65
  .no_proxy()
61
66
  .build()
62
67
  .unwrap();
63
68
 
64
- // never returns
65
- let url = "http://192.0.2.1:81/slow";
69
+ let url = format!("http://{}", server.addr());
66
70
 
67
71
  let res = client.get(url).send().await;
68
72