wreq-rb 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +1 -1
- data/README.md +30 -3
- data/exe/wreq +211 -0
- data/ext/wreq_rb/Cargo.toml +1 -1
- data/ext/wreq_rb/src/client.rs +115 -4
- data/lib/wreq-rb/version.rb +1 -1
- data/patches/0002-add-cancel-connections.patch +181 -0
- metadata +7 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e073dd8e896b2e9a72abc5d34c18fbdc22e70281f711809d48b927821371a88d
|
|
4
|
+
data.tar.gz: a9c9c86618a68e46c007b518587c57c057603312f6a9fcf276fa2dc075c919a8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6b04dc928f331472dc4ba7c597a4070952ee49900853134795a87f29527c1aa59418479ead50a8f4684f79854ae9b17f2c4ebbfe8c8ab1e9a509233f3936d53d
|
|
7
|
+
data.tar.gz: c105d0bec620ad3975020a5228950b4fe733e7f5314bbd78c8ea94747b35929b6a299340f866cbda41c62b3a6c2b98f57dfdcacd71d80f7eb5df1a768d138f5a
|
data/Cargo.lock
CHANGED
data/README.md
CHANGED
|
@@ -77,9 +77,23 @@ client = Wreq::Client.new(
|
|
|
77
77
|
zstd: true, # enable zstd decompression
|
|
78
78
|
emulation: "chrome_143", # browser emulation (enabled by default)
|
|
79
79
|
emulation_os: "windows", # OS emulation: windows, macos (default), linux, android, ios
|
|
80
|
+
header_order: [ # wire order of headers (names only, case-sensitive)
|
|
81
|
+
"host", # listed headers appear first in the given order, remaining
|
|
82
|
+
"user-agent", # emulation headers follow.
|
|
83
|
+
"accept",
|
|
84
|
+
],
|
|
80
85
|
headers: { # default headers for all requests
|
|
81
86
|
"Accept" => "application/json"
|
|
82
|
-
}
|
|
87
|
+
},
|
|
88
|
+
referer: true, # auto-set Referer header on redirects (default: true)
|
|
89
|
+
pool_max_idle_per_host: 10, # max idle connections per host
|
|
90
|
+
pool_max_size: 100, # max total connections in the pool
|
|
91
|
+
tcp_nodelay: true, # disable Nagle algorithm (default: true)
|
|
92
|
+
tcp_keepalive: 15, # SO_KEEPALIVE interval in seconds (default: 15)
|
|
93
|
+
local_address: "1.2.3.4", # bind outgoing connections to this source IP
|
|
94
|
+
tls_sni: true, # send SNI in TLS handshake (default: true)
|
|
95
|
+
min_tls_version: "tls1.2", # minimum TLS version: tls1.0, tls1.1, tls1.2, tls1.3
|
|
96
|
+
max_tls_version: "tls1.3", # maximum TLS version
|
|
83
97
|
)
|
|
84
98
|
|
|
85
99
|
resp = client.get("https://api.example.com/data")
|
|
@@ -100,6 +114,19 @@ All methods are available on both `Wreq` (module-level) and `Wreq::Client` (inst
|
|
|
100
114
|
| `head(url, **opts)` | HEAD request |
|
|
101
115
|
| `options(url, **opts)` | OPTIONS request |
|
|
102
116
|
|
|
117
|
+
### Cancelling Requests
|
|
118
|
+
|
|
119
|
+
Call `cancel` on a client to abort all in-flight requests and close their underlying connections immediately:
|
|
120
|
+
|
|
121
|
+
```ruby
|
|
122
|
+
client = Wreq::Client.new
|
|
123
|
+
|
|
124
|
+
# From another thread:
|
|
125
|
+
t = Thread.new { client.get("https://slow.example.com/big-download") }
|
|
126
|
+
sleep 1
|
|
127
|
+
client.cancel # all in-flight requests return with "request interrupted" error
|
|
128
|
+
```
|
|
129
|
+
|
|
103
130
|
### Per-Request Options
|
|
104
131
|
|
|
105
132
|
Pass an options hash as the second argument to any HTTP method:
|
|
@@ -128,7 +155,7 @@ resp = Wreq.get("https://tls.peet.ws/api/all")
|
|
|
128
155
|
|
|
129
156
|
# Explicit browser emulation
|
|
130
157
|
client = Wreq::Client.new(emulation: "firefox_146")
|
|
131
|
-
client = Wreq::Client.new(emulation: "
|
|
158
|
+
client = Wreq::Client.new(emulation: "safari_18.5")
|
|
132
159
|
client = Wreq::Client.new(emulation: "edge_142")
|
|
133
160
|
|
|
134
161
|
# Disable emulation entirely
|
|
@@ -142,7 +169,7 @@ client = Wreq::Client.new(emulation: "chrome_145", emulation_os: "linux")
|
|
|
142
169
|
client = Wreq::Client.new(emulation: "chrome_143", user_agent: "MyBot/1.0")
|
|
143
170
|
|
|
144
171
|
# Per-request emulation override
|
|
145
|
-
resp = client.get("https://example.com", emulation: "
|
|
172
|
+
resp = client.get("https://example.com", emulation: "safari_26.2")
|
|
146
173
|
```
|
|
147
174
|
|
|
148
175
|
### Supported Browsers
|
data/exe/wreq
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "optparse"
|
|
5
|
+
require "json"
|
|
6
|
+
require "wreq-rb"
|
|
7
|
+
|
|
8
|
+
cfg = {
|
|
9
|
+
method: nil,
|
|
10
|
+
headers: {},
|
|
11
|
+
include: false,
|
|
12
|
+
silent: false,
|
|
13
|
+
redirect: 10,
|
|
14
|
+
verify_cert: true,
|
|
15
|
+
verify_host: true,
|
|
16
|
+
pretty: false,
|
|
17
|
+
}
|
|
18
|
+
req = {}
|
|
19
|
+
|
|
20
|
+
parser = OptionParser.new do |o|
|
|
21
|
+
o.banner = "Usage: wreq [METHOD] URL [options]"
|
|
22
|
+
|
|
23
|
+
o.separator ""
|
|
24
|
+
o.separator "Request:"
|
|
25
|
+
|
|
26
|
+
o.on("-X", "--request METHOD", "HTTP method (default: GET)") { |m| cfg[:method] = m.upcase }
|
|
27
|
+
|
|
28
|
+
o.on("-H", "--header LINE", 'Request header, e.g. "Accept: application/json" (repeatable)') do |h|
|
|
29
|
+
name, _, value = h.partition(":")
|
|
30
|
+
cfg[:headers][name.strip] = value.strip
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
o.on("-d", "--data DATA",
|
|
34
|
+
"Request body. Prefix with @ to read from file, e.g. @body.json") do |d|
|
|
35
|
+
cfg[:method] ||= "POST"
|
|
36
|
+
if d.start_with?("@")
|
|
37
|
+
path = d[1..]
|
|
38
|
+
abort "wreq: cannot read '#{path}': file not found" unless File.exist?(path)
|
|
39
|
+
req[:body] = File.binread(path)
|
|
40
|
+
else
|
|
41
|
+
req[:body] = d
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
o.on("--json DATA",
|
|
46
|
+
"JSON body (sets Content-Type: application/json). Prefix with @ for file.") do |d|
|
|
47
|
+
cfg[:method] ||= "POST"
|
|
48
|
+
raw = d.start_with?("@") ? File.read(d[1..]) : d
|
|
49
|
+
begin
|
|
50
|
+
req[:json] = JSON.parse(raw)
|
|
51
|
+
rescue JSON::ParserError => e
|
|
52
|
+
abort "wreq: invalid JSON: #{e.message}"
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
o.on("--form PAIR", "Form field as key=value (repeatable)") do |f|
|
|
57
|
+
cfg[:method] ||= "POST"
|
|
58
|
+
key, _, val = f.partition("=")
|
|
59
|
+
(req[:form] ||= {})[key] = val
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
o.on("-q", "--query PAIR", "Query parameter as key=value (repeatable)") do |q|
|
|
63
|
+
key, _, val = q.partition("=")
|
|
64
|
+
(req[:query] ||= {})[key] = val
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
o.separator ""
|
|
68
|
+
o.separator "Auth:"
|
|
69
|
+
|
|
70
|
+
o.on("-u", "--user USER[:PASS]", "Basic auth credentials") do |u|
|
|
71
|
+
user, _, pass = u.partition(":")
|
|
72
|
+
req[:basic] = [user, pass]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
o.on("--bearer TOKEN", "Bearer token auth") { |t| req[:bearer] = t }
|
|
76
|
+
|
|
77
|
+
o.separator ""
|
|
78
|
+
o.separator "Output:"
|
|
79
|
+
|
|
80
|
+
o.on("-i", "--include", "Include response headers in stdout before the body") do
|
|
81
|
+
cfg[:include] = true
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
o.on("-s", "--silent", "Suppress body output") { cfg[:silent] = true }
|
|
85
|
+
|
|
86
|
+
o.on("-o", "--output FILE", "Write body to FILE") { |f| cfg[:output] = f }
|
|
87
|
+
|
|
88
|
+
o.on("--pretty", "Pretty-print JSON response bodies") { cfg[:pretty] = true }
|
|
89
|
+
|
|
90
|
+
o.separator ""
|
|
91
|
+
o.separator "Connection:"
|
|
92
|
+
|
|
93
|
+
o.on("-L", "--location", "Follow redirects (default: on, max 10)") { cfg[:redirect] = 10 }
|
|
94
|
+
o.on("--no-location", "Do not follow redirects") { cfg[:redirect] = false }
|
|
95
|
+
o.on("--max-redirects N", Integer, "Max redirects (default: 10)") { |n| cfg[:redirect] = n }
|
|
96
|
+
|
|
97
|
+
o.on("-k", "--insecure", "Skip TLS certificate verification") do
|
|
98
|
+
cfg[:verify_cert] = false
|
|
99
|
+
cfg[:verify_host] = false
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
o.on("-x", "--proxy URL", "Proxy URL (e.g. http://host:port)") { |p| cfg[:proxy] = p }
|
|
103
|
+
|
|
104
|
+
o.on("--local-address IP", "Bind outgoing connections to this source IP address") do |ip|
|
|
105
|
+
cfg[:local_address] = ip
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
o.on("--timeout SECS", Float, "Total timeout in seconds") { |t| cfg[:timeout] = t }
|
|
109
|
+
o.on("--connect-timeout SECS", Float, "Connection timeout in seconds") { |t| cfg[:connect_timeout] = t }
|
|
110
|
+
|
|
111
|
+
o.on("--http1", "Force HTTP/1.1") { cfg[:http1_only] = true }
|
|
112
|
+
o.on("--http2", "Force HTTP/2") { cfg[:http2_only] = true }
|
|
113
|
+
|
|
114
|
+
o.separator ""
|
|
115
|
+
o.separator "Emulation:"
|
|
116
|
+
|
|
117
|
+
o.on("--emulation PROFILE",
|
|
118
|
+
"Browser emulation profile (e.g. chrome_145, firefox_147, safari_26.2)") do |e|
|
|
119
|
+
cfg[:emulation] = e
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
o.on("--emulation-os OS",
|
|
123
|
+
"Emulation OS override: windows, macos, linux, android, ios") do |os|
|
|
124
|
+
cfg[:emulation_os] = os
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
o.on("--no-emulation", "Disable browser emulation entirely") { cfg[:emulation] = false }
|
|
128
|
+
|
|
129
|
+
o.on("-A", "--user-agent UA", "Override User-Agent header") { |ua| cfg[:user_agent] = ua }
|
|
130
|
+
|
|
131
|
+
o.on("--cookie-store", "Enable persistent cookie jar for this request") { cfg[:cookie_store] = true }
|
|
132
|
+
|
|
133
|
+
o.separator ""
|
|
134
|
+
|
|
135
|
+
o.on("-V", "--version", "Print version and exit") { puts "wreq #{Wreq::VERSION}"; exit 0 }
|
|
136
|
+
o.on_tail("-h", "--help", "Show this help") { puts o; exit 0 }
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
args = ARGV.dup
|
|
140
|
+
|
|
141
|
+
# Allow bare method as first positional arg: `wreq POST https://...`
|
|
142
|
+
if args.first =~ /\A[A-Z]+\z/
|
|
143
|
+
cfg[:method] = args.shift
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
begin
|
|
147
|
+
parser.parse!(args)
|
|
148
|
+
rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
|
|
149
|
+
abort "wreq: #{e.message}\nRun 'wreq --help' for usage."
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
url = args.shift
|
|
153
|
+
unless url
|
|
154
|
+
$stderr.puts parser
|
|
155
|
+
exit 1
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
cfg[:method] ||= "GET"
|
|
159
|
+
|
|
160
|
+
# Build Wreq::Client options (connection/emulation settings only)
|
|
161
|
+
CLIENT_KEYS = %i[emulation emulation_os user_agent proxy local_address timeout connect_timeout
|
|
162
|
+
http1_only http2_only cookie_store https_only].freeze
|
|
163
|
+
client_opts = cfg.slice(*CLIENT_KEYS).reject { |_, v| v.nil? }
|
|
164
|
+
client_opts[:redirect] = cfg[:redirect]
|
|
165
|
+
client_opts[:verify_cert] = cfg[:verify_cert]
|
|
166
|
+
client_opts[:verify_host] = cfg[:verify_host]
|
|
167
|
+
|
|
168
|
+
client = Wreq::Client.new(**client_opts)
|
|
169
|
+
|
|
170
|
+
# Per-request headers
|
|
171
|
+
req[:headers] = cfg[:headers] unless cfg[:headers].empty?
|
|
172
|
+
|
|
173
|
+
# Execute request
|
|
174
|
+
begin
|
|
175
|
+
response = client.public_send(cfg[:method].downcase, url, **req)
|
|
176
|
+
rescue NoMethodError
|
|
177
|
+
abort "wreq: unsupported HTTP method '#{cfg[:method]}'"
|
|
178
|
+
rescue => e
|
|
179
|
+
abort "wreq: #{e.message}"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# -i: print response status + headers to stdout before body
|
|
183
|
+
if cfg[:include]
|
|
184
|
+
puts "#{response.version} #{response.status}"
|
|
185
|
+
response.headers.each do |k, values|
|
|
186
|
+
Array(values).each { |v| puts "#{k}: #{v}" }
|
|
187
|
+
end
|
|
188
|
+
puts ""
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Output body
|
|
192
|
+
unless cfg[:silent] || cfg[:method] == "HEAD"
|
|
193
|
+
if cfg[:output]
|
|
194
|
+
bytes = response.body_bytes.pack("C*")
|
|
195
|
+
File.binwrite(cfg[:output], bytes)
|
|
196
|
+
warn "Saved #{bytes.bytesize} bytes to #{cfg[:output]}"
|
|
197
|
+
else
|
|
198
|
+
body = response.body
|
|
199
|
+
if cfg[:pretty] && !body.nil? && !body.empty?
|
|
200
|
+
body = begin
|
|
201
|
+
JSON.pretty_generate(JSON.parse(body))
|
|
202
|
+
rescue JSON::ParserError
|
|
203
|
+
body
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
print body unless body.nil?
|
|
207
|
+
$stdout.puts if body && !body.empty? && !body.end_with?("\n")
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
exit response.success? ? 0 : 1
|
data/ext/wreq_rb/Cargo.toml
CHANGED
data/ext/wreq_rb/src/client.rs
CHANGED
|
@@ -10,7 +10,9 @@ use magnus::{
|
|
|
10
10
|
};
|
|
11
11
|
use tokio::runtime::Runtime;
|
|
12
12
|
use tokio_util::sync::CancellationToken;
|
|
13
|
-
use
|
|
13
|
+
use std::net::IpAddr;
|
|
14
|
+
use wreq::header::{HeaderMap, HeaderName, HeaderValue, OrigHeaderMap};
|
|
15
|
+
use wreq::tls::TlsVersion;
|
|
14
16
|
use wreq_util::{Emulation as BrowserEmulation, EmulationOS, EmulationOption};
|
|
15
17
|
|
|
16
18
|
use crate::error::{generic_error, to_magnus_error};
|
|
@@ -178,6 +180,7 @@ fn build_emulation_option(
|
|
|
178
180
|
#[magnus::wrap(class = "Wreq::Client", free_immediately)]
|
|
179
181
|
struct Client {
|
|
180
182
|
inner: wreq::Client,
|
|
183
|
+
cancel_token: std::sync::Mutex<CancellationToken>,
|
|
181
184
|
}
|
|
182
185
|
|
|
183
186
|
impl Client {
|
|
@@ -192,6 +195,12 @@ impl Client {
|
|
|
192
195
|
let mut builder = wreq::Client::builder();
|
|
193
196
|
|
|
194
197
|
if let Some(opts) = opts {
|
|
198
|
+
// Apply header_order BEFORE emulation so the user's ordering takes precedence
|
|
199
|
+
if let Some(ary) = hash_get_array(&opts, "header_order")? {
|
|
200
|
+
let orig = array_to_orig_header_map(ary)?;
|
|
201
|
+
builder = builder.orig_headers(orig);
|
|
202
|
+
}
|
|
203
|
+
|
|
195
204
|
if let Some(val) = hash_get_value(&opts, "emulation")? {
|
|
196
205
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
197
206
|
if val.is_kind_of(ruby.class_false_class()) {
|
|
@@ -293,12 +302,50 @@ impl Client {
|
|
|
293
302
|
if let Some(v) = hash_get_bool(&opts, "zstd")? {
|
|
294
303
|
builder = builder.zstd(v);
|
|
295
304
|
}
|
|
305
|
+
|
|
306
|
+
if let Some(v) = hash_get_bool(&opts, "referer")? {
|
|
307
|
+
builder = builder.referer(v);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
if let Some(n) = hash_get_usize(&opts, "pool_max_idle_per_host")? {
|
|
311
|
+
builder = builder.pool_max_idle_per_host(n);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
if let Some(n) = hash_get_u32(&opts, "pool_max_size")? {
|
|
315
|
+
builder = builder.pool_max_size(n);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
if let Some(v) = hash_get_bool(&opts, "tcp_nodelay")? {
|
|
319
|
+
builder = builder.tcp_nodelay(v);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
if let Some(t) = hash_get_float(&opts, "tcp_keepalive")? {
|
|
323
|
+
builder = builder.tcp_keepalive(Duration::from_secs_f64(t));
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
if let Some(addr_str) = hash_get_string(&opts, "local_address")? {
|
|
327
|
+
let addr: IpAddr = addr_str.parse()
|
|
328
|
+
.map_err(|_| generic_error(format!("invalid IP address: '{}'", addr_str)))?;
|
|
329
|
+
builder = builder.local_address(addr);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
if let Some(v) = hash_get_bool(&opts, "tls_sni")? {
|
|
333
|
+
builder = builder.tls_sni(v);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if let Some(s) = hash_get_string(&opts, "min_tls_version")? {
|
|
337
|
+
builder = builder.min_tls_version(parse_tls_version(&s)?);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if let Some(s) = hash_get_string(&opts, "max_tls_version")? {
|
|
341
|
+
builder = builder.max_tls_version(parse_tls_version(&s)?);
|
|
342
|
+
}
|
|
296
343
|
} else {
|
|
297
344
|
builder = builder.emulation(DEFAULT_EMULATION);
|
|
298
345
|
}
|
|
299
346
|
|
|
300
347
|
let client = builder.build().map_err(to_magnus_error)?;
|
|
301
|
-
Ok(Client { inner: client })
|
|
348
|
+
Ok(Client { inner: client, cancel_token: std::sync::Mutex::new(CancellationToken::new()) })
|
|
302
349
|
}
|
|
303
350
|
|
|
304
351
|
/// client.get(url) or client.get(url, opts)
|
|
@@ -330,6 +377,19 @@ impl Client {
|
|
|
330
377
|
self.execute_method("OPTIONS", args)
|
|
331
378
|
}
|
|
332
379
|
|
|
380
|
+
fn cancel(&self) {
|
|
381
|
+
// Replace the cancel token first so new requests use a fresh token,
|
|
382
|
+
// then cancel the old one to unblock all current in-flight select!s.
|
|
383
|
+
let old_token = {
|
|
384
|
+
let mut guard = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner());
|
|
385
|
+
let old = guard.clone();
|
|
386
|
+
*guard = CancellationToken::new();
|
|
387
|
+
old
|
|
388
|
+
};
|
|
389
|
+
old_token.cancel();
|
|
390
|
+
self.inner.cancel_connections();
|
|
391
|
+
}
|
|
392
|
+
|
|
333
393
|
fn execute_method(&self, method_str: &str, args: &[Value]) -> Result<Response, magnus::Error> {
|
|
334
394
|
let url: String = if args.is_empty() {
|
|
335
395
|
return Err(generic_error("url is required"));
|
|
@@ -353,13 +413,16 @@ impl Client {
|
|
|
353
413
|
req = apply_request_options(req, &opts)?;
|
|
354
414
|
}
|
|
355
415
|
|
|
416
|
+
let client_token = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
|
417
|
+
|
|
356
418
|
// Release the GVL so other Ruby threads can run during I/O.
|
|
357
419
|
let outcome: RequestOutcome = unsafe {
|
|
358
|
-
without_gvl(|
|
|
420
|
+
without_gvl(|thread_token| {
|
|
359
421
|
runtime().block_on(async {
|
|
360
422
|
tokio::select! {
|
|
361
423
|
biased;
|
|
362
|
-
_ =
|
|
424
|
+
_ = thread_token.cancelled() => RequestOutcome::Interrupted,
|
|
425
|
+
_ = client_token.cancelled() => RequestOutcome::Interrupted,
|
|
363
426
|
res = execute_request(req) => match res {
|
|
364
427
|
Ok(data) => RequestOutcome::Ok(data),
|
|
365
428
|
Err(e) => RequestOutcome::Err(e),
|
|
@@ -529,6 +592,32 @@ fn hash_get_bool(hash: &RHash, key: &str) -> Result<Option<bool>, magnus::Error>
|
|
|
529
592
|
}
|
|
530
593
|
}
|
|
531
594
|
|
|
595
|
+
fn hash_get_usize(hash: &RHash, key: &str) -> Result<Option<usize>, magnus::Error> {
|
|
596
|
+
match hash_get_value(hash, key)? {
|
|
597
|
+
Some(v) => Ok(Some(TryConvert::try_convert(v)?)),
|
|
598
|
+
None => Ok(None),
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
fn hash_get_u32(hash: &RHash, key: &str) -> Result<Option<u32>, magnus::Error> {
|
|
603
|
+
match hash_get_value(hash, key)? {
|
|
604
|
+
Some(v) => Ok(Some(TryConvert::try_convert(v)?)),
|
|
605
|
+
None => Ok(None),
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
fn parse_tls_version(s: &str) -> Result<TlsVersion, magnus::Error> {
|
|
610
|
+
match s {
|
|
611
|
+
"tls1.0" | "tls_1_0" | "1.0" => Ok(TlsVersion::TLS_1_0),
|
|
612
|
+
"tls1.1" | "tls_1_1" | "1.1" => Ok(TlsVersion::TLS_1_1),
|
|
613
|
+
"tls1.2" | "tls_1_2" | "1.2" => Ok(TlsVersion::TLS_1_2),
|
|
614
|
+
"tls1.3" | "tls_1_3" | "1.3" => Ok(TlsVersion::TLS_1_3),
|
|
615
|
+
_ => Err(generic_error(format!(
|
|
616
|
+
"unknown TLS version '{}'. Use: 'tls1.2', 'tls1.3'", s
|
|
617
|
+
))),
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
532
621
|
fn hash_get_hash(hash: &RHash, key: &str) -> Result<Option<RHash>, magnus::Error> {
|
|
533
622
|
match hash_get_value(hash, key)? {
|
|
534
623
|
Some(v) => Ok(Some(RHash::try_convert(v)?)),
|
|
@@ -536,6 +625,27 @@ fn hash_get_hash(hash: &RHash, key: &str) -> Result<Option<RHash>, magnus::Error
|
|
|
536
625
|
}
|
|
537
626
|
}
|
|
538
627
|
|
|
628
|
+
fn hash_get_array(hash: &RHash, key: &str) -> Result<Option<RArray>, magnus::Error> {
|
|
629
|
+
match hash_get_value(hash, key)? {
|
|
630
|
+
Some(v) => Ok(Some(RArray::try_convert(v)?)),
|
|
631
|
+
None => Ok(None),
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
fn array_to_orig_header_map(ary: RArray) -> Result<OrigHeaderMap, magnus::Error> {
|
|
636
|
+
let mut orig = OrigHeaderMap::with_capacity(ary.len());
|
|
637
|
+
for elem in ary.into_iter() {
|
|
638
|
+
let ruby = unsafe { Ruby::get_unchecked() };
|
|
639
|
+
let name_str: String = if elem.is_kind_of(ruby.class_symbol()) {
|
|
640
|
+
elem.funcall("to_s", ())?
|
|
641
|
+
} else {
|
|
642
|
+
TryConvert::try_convert(elem)?
|
|
643
|
+
};
|
|
644
|
+
orig.insert(name_str);
|
|
645
|
+
}
|
|
646
|
+
Ok(orig)
|
|
647
|
+
}
|
|
648
|
+
|
|
539
649
|
fn hash_to_header_map(hash: &RHash) -> Result<HeaderMap, magnus::Error> {
|
|
540
650
|
let mut hmap = HeaderMap::new();
|
|
541
651
|
hash.foreach(|k: Value, v: Value| {
|
|
@@ -589,6 +699,7 @@ pub fn init(_ruby: &magnus::Ruby, module: &magnus::RModule) -> Result<(), magnus
|
|
|
589
699
|
client_class.define_method("delete", method!(Client::delete, -1))?;
|
|
590
700
|
client_class.define_method("head", method!(Client::head, -1))?;
|
|
591
701
|
client_class.define_method("options", method!(Client::options, -1))?;
|
|
702
|
+
client_class.define_method("cancel", method!(Client::cancel, 0))?;
|
|
592
703
|
|
|
593
704
|
module.define_module_function("get", function!(wreq_get, -1))?;
|
|
594
705
|
module.define_module_function("post", function!(wreq_post, -1))?;
|
data/lib/wreq-rb/version.rb
CHANGED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
diff --git a/src/client/http.rs b/src/client/http.rs
|
|
2
|
+
index 462c77d0..6231e700 100644
|
|
3
|
+
--- a/src/client/http.rs
|
|
4
|
+
+++ b/src/client/http.rs
|
|
5
|
+
@@ -171,6 +171,7 @@ type ClientRef = Either<ClientService, BoxedClientService>;
|
|
6
|
+
#[derive(Clone)]
|
|
7
|
+
pub struct Client {
|
|
8
|
+
inner: Arc<ClientRef>,
|
|
9
|
+
+ cancel_connections: Arc<dyn Fn() + Send + Sync>,
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/// A [`ClientBuilder`] can be used to create a [`Client`] with custom configuration.
|
|
13
|
+
@@ -439,6 +440,14 @@ impl Client {
|
|
14
|
+
let fut = Oneshot::new(self.inner.as_ref().clone(), req);
|
|
15
|
+
Pending::request(uri, fut)
|
|
16
|
+
}
|
|
17
|
+
+
|
|
18
|
+
+ /// Cancel all background connection tasks immediately.
|
|
19
|
+
+ ///
|
|
20
|
+
+ /// This force-drops every tracked H1/H2 connection task, causing the
|
|
21
|
+
+ /// underlying TCP socket to close abruptly.
|
|
22
|
+
+ pub fn cancel_connections(&self) {
|
|
23
|
+
+ (self.cancel_connections)()
|
|
24
|
+
+ }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
impl tower::Service<Request> for Client {
|
|
28
|
+
@@ -495,7 +504,7 @@ impl ClientBuilder {
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Create base client service
|
|
32
|
+
- let service = {
|
|
33
|
+
+ let (service, cancel_connections) = {
|
|
34
|
+
let (tls_options, http1_options, http2_options) = config.transport_options.into();
|
|
35
|
+
|
|
36
|
+
let resolver = {
|
|
37
|
+
@@ -555,7 +564,7 @@ impl ClientBuilder {
|
|
38
|
+
.build(config.connector_layers)?;
|
|
39
|
+
|
|
40
|
+
// Build client
|
|
41
|
+
- HttpClient::builder(TokioExecutor::new())
|
|
42
|
+
+ let http_client = HttpClient::builder(TokioExecutor::new())
|
|
43
|
+
.http1_options(http1_options)
|
|
44
|
+
.http2_options(http2_options)
|
|
45
|
+
.http2_only(matches!(config.http_version_pref, HttpVersionPref::Http2))
|
|
46
|
+
@@ -564,8 +573,10 @@ impl ClientBuilder {
|
|
47
|
+
.pool_idle_timeout(config.pool_idle_timeout)
|
|
48
|
+
.pool_max_idle_per_host(config.pool_max_idle_per_host)
|
|
49
|
+
.pool_max_size(config.pool_max_size)
|
|
50
|
+
- .build(connector)
|
|
51
|
+
- .map_err(Into::into as _)
|
|
52
|
+
+ .build(connector);
|
|
53
|
+
+ // Capture abort fn before tower layers consume the client
|
|
54
|
+
+ let cancel_connections = http_client.cancel_connections_fn();
|
|
55
|
+
+ (http_client.map_err(Into::into as _), cancel_connections)
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
// Configured client service with layers
|
|
59
|
+
@@ -633,6 +644,7 @@ impl ClientBuilder {
|
|
60
|
+
|
|
61
|
+
Ok(Client {
|
|
62
|
+
inner: Arc::new(client),
|
|
63
|
+
+ cancel_connections,
|
|
64
|
+
})
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
diff --git a/src/client/http/client.rs b/src/client/http/client.rs
|
|
68
|
+
index a29c52dd..7f6ce68f 100644
|
|
69
|
+
--- a/src/client/http/client.rs
|
|
70
|
+
+++ b/src/client/http/client.rs
|
|
71
|
+
@@ -10,7 +10,7 @@ use std::{
|
|
72
|
+
future::Future,
|
|
73
|
+
num::NonZeroU32,
|
|
74
|
+
pin::Pin,
|
|
75
|
+
- sync::Arc,
|
|
76
|
+
+ sync::{Arc, Mutex},
|
|
77
|
+
task::{self, Poll},
|
|
78
|
+
time::Duration,
|
|
79
|
+
};
|
|
80
|
+
@@ -119,6 +119,7 @@ pub struct HttpClient<C, B> {
|
|
81
|
+
h1_builder: conn::http1::Builder,
|
|
82
|
+
h2_builder: conn::http2::Builder<Exec>,
|
|
83
|
+
pool: pool::Pool<PoolClient<B>, ConnectIdentity>,
|
|
84
|
+
+ conn_abort_handles: Arc<Mutex<Vec<tokio::task::AbortHandle>>>,
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
#[derive(Clone, Copy)]
|
|
88
|
+
@@ -452,8 +453,8 @@ where
|
|
89
|
+
+ Send
|
|
90
|
+
+ Unpin
|
|
91
|
+
+ 'static {
|
|
92
|
+
- let executor = self.exec.clone();
|
|
93
|
+
let pool = self.pool.clone();
|
|
94
|
+
+ let conn_abort_handles = self.conn_abort_handles.clone();
|
|
95
|
+
|
|
96
|
+
let h1_builder = self.h1_builder.clone();
|
|
97
|
+
let h2_builder = self.h2_builder.clone();
|
|
98
|
+
@@ -513,10 +514,17 @@ where
|
|
99
|
+
trace!(
|
|
100
|
+
"http2 handshake complete, spawning background dispatcher task"
|
|
101
|
+
);
|
|
102
|
+
- executor.execute(
|
|
103
|
+
- conn.map_err(|_e| debug!("client connection error: {}", _e))
|
|
104
|
+
- .map(|_| ()),
|
|
105
|
+
- );
|
|
106
|
+
+ let h2_conn_future = conn
|
|
107
|
+
+ .map_err(|_e| debug!("client connection error: {}", _e))
|
|
108
|
+
+ .map(|_| ());
|
|
109
|
+
+ let abort_handle = tokio::spawn(h2_conn_future).abort_handle();
|
|
110
|
+
+ {
|
|
111
|
+
+ let mut handles = conn_abort_handles.lock().unwrap_or_else(|e| e.into_inner());
|
|
112
|
+
+ if handles.len() >= 64 {
|
|
113
|
+
+ handles.retain(|h| !h.is_finished());
|
|
114
|
+
+ }
|
|
115
|
+
+ handles.push(abort_handle);
|
|
116
|
+
+ }
|
|
117
|
+
|
|
118
|
+
// Wait for 'conn' to ready up before we
|
|
119
|
+
// declare this tx as usable
|
|
120
|
+
@@ -544,8 +552,7 @@ where
|
|
121
|
+
// Spawn the connection task in the background using the executor.
|
|
122
|
+
// The task manages the HTTP/1.1 connection, including upgrades (e.g., WebSocket).
|
|
123
|
+
// Errors are sent via err_tx to ensure they can be checked if the sender (tx) fails.
|
|
124
|
+
- executor.execute(
|
|
125
|
+
- conn.with_upgrades()
|
|
126
|
+
+ let h1_conn_future = conn.with_upgrades()
|
|
127
|
+
.map_err(|e| {
|
|
128
|
+
// Log the connection error at debug level for diagnostic purposes.
|
|
129
|
+
debug!("client connection error: {:?}", e);
|
|
130
|
+
@@ -555,8 +562,15 @@ where
|
|
131
|
+
// (e.g., if the receiver is dropped, which is handled later).
|
|
132
|
+
let _ = err_tx.send(e);
|
|
133
|
+
})
|
|
134
|
+
- .map(|_| ()),
|
|
135
|
+
- );
|
|
136
|
+
+ .map(|_| ());
|
|
137
|
+
+ let abort_handle = tokio::spawn(h1_conn_future).abort_handle();
|
|
138
|
+
+ {
|
|
139
|
+
+ let mut handles = conn_abort_handles.lock().unwrap_or_else(|e| e.into_inner());
|
|
140
|
+
+ if handles.len() >= 64 {
|
|
141
|
+
+ handles.retain(|h| !h.is_finished());
|
|
142
|
+
+ }
|
|
143
|
+
+ handles.push(abort_handle);
|
|
144
|
+
+ }
|
|
145
|
+
|
|
146
|
+
// Log that the client is waiting for the connection to be ready.
|
|
147
|
+
// Readiness indicates the sender (tx) can accept a request without blocking. More actions
|
|
148
|
+
@@ -682,10 +696,25 @@ impl<C: Clone, B> Clone for HttpClient<C, B> {
|
|
149
|
+
h2_builder: self.h2_builder.clone(),
|
|
150
|
+
connector: self.connector.clone(),
|
|
151
|
+
pool: self.pool.clone(),
|
|
152
|
+
+ conn_abort_handles: self.conn_abort_handles.clone(),
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
+impl<C, B> HttpClient<C, B> {
|
|
158
|
+
+ /// Returns a closure that cancels all tracked connection tasks when called.
|
|
159
|
+
+ /// Each call drains the handle list and aborts every stored task.
|
|
160
|
+
+ pub fn cancel_connections_fn(&self) -> Arc<dyn Fn() + Send + Sync + 'static> {
|
|
161
|
+
+ let handles = self.conn_abort_handles.clone();
|
|
162
|
+
+ Arc::new(move || {
|
|
163
|
+
+ let mut guard = handles.lock().unwrap_or_else(|e| e.into_inner());
|
|
164
|
+
+ for handle in guard.drain(..) {
|
|
165
|
+
+ handle.abort();
|
|
166
|
+
+ }
|
|
167
|
+
+ })
|
|
168
|
+
+ }
|
|
169
|
+
+}
|
|
170
|
+
+
|
|
171
|
+
/// A pooled HTTP connection that can send requests
|
|
172
|
+
struct PoolClient<B> {
|
|
173
|
+
conn_info: Connected,
|
|
174
|
+
@@ -998,6 +1027,7 @@ impl Builder {
|
|
175
|
+
h2_builder: self.h2_builder,
|
|
176
|
+
connector,
|
|
177
|
+
pool: pool::Pool::new(self.pool_config, exec, timer),
|
|
178
|
+
+ conn_abort_handles: Arc::new(Mutex::new(Vec::new())),
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
metadata
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: wreq-rb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yicheng Zhou
|
|
8
8
|
- Illia Zub
|
|
9
9
|
autorequire:
|
|
10
|
-
bindir:
|
|
10
|
+
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2026-03-
|
|
12
|
+
date: 2026-03-17 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: rb_sys
|
|
@@ -29,7 +29,8 @@ description: An ergonomic Ruby HTTP client powered by Rust's wreq library, featu
|
|
|
29
29
|
TLS fingerprint emulation (JA3/JA4), HTTP/2 support, cookie handling, proxy support,
|
|
30
30
|
and redirect policies.
|
|
31
31
|
email:
|
|
32
|
-
executables:
|
|
32
|
+
executables:
|
|
33
|
+
- wreq
|
|
33
34
|
extensions:
|
|
34
35
|
- ext/wreq_rb/extconf.rb
|
|
35
36
|
extra_rdoc_files: []
|
|
@@ -37,6 +38,7 @@ files:
|
|
|
37
38
|
- Cargo.lock
|
|
38
39
|
- Cargo.toml
|
|
39
40
|
- README.md
|
|
41
|
+
- exe/wreq
|
|
40
42
|
- ext/wreq_rb/Cargo.toml
|
|
41
43
|
- ext/wreq_rb/extconf.rb
|
|
42
44
|
- ext/wreq_rb/src/client.rs
|
|
@@ -46,6 +48,7 @@ files:
|
|
|
46
48
|
- lib/wreq-rb.rb
|
|
47
49
|
- lib/wreq-rb/version.rb
|
|
48
50
|
- patches/0001-add-transfer-size-tracking.patch
|
|
51
|
+
- patches/0002-add-cancel-connections.patch
|
|
49
52
|
- vendor/wreq/Cargo.toml
|
|
50
53
|
- vendor/wreq/LICENSE
|
|
51
54
|
- vendor/wreq/README.md
|