wreq-rb 0.3.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +1 -1
- data/README.md +36 -3
- data/exe/wreq +211 -0
- data/ext/wreq_rb/Cargo.toml +2 -2
- data/ext/wreq_rb/src/client.rs +148 -10
- data/lib/wreq-rb/version.rb +1 -1
- data/patches/0002-add-cancel-connections.patch +181 -0
- metadata +7 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e073dd8e896b2e9a72abc5d34c18fbdc22e70281f711809d48b927821371a88d
|
|
4
|
+
data.tar.gz: a9c9c86618a68e46c007b518587c57c057603312f6a9fcf276fa2dc075c919a8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6b04dc928f331472dc4ba7c597a4070952ee49900853134795a87f29527c1aa59418479ead50a8f4684f79854ae9b17f2c4ebbfe8c8ab1e9a509233f3936d53d
|
|
7
|
+
data.tar.gz: c105d0bec620ad3975020a5228950b4fe733e7f5314bbd78c8ea94747b35929b6a299340f866cbda41c62b3a6c2b98f57dfdcacd71d80f7eb5df1a768d138f5a
|
data/Cargo.lock
CHANGED
data/README.md
CHANGED
|
@@ -76,9 +76,24 @@ client = Wreq::Client.new(
|
|
|
76
76
|
deflate: true, # enable deflate decompression
|
|
77
77
|
zstd: true, # enable zstd decompression
|
|
78
78
|
emulation: "chrome_143", # browser emulation (enabled by default)
|
|
79
|
+
emulation_os: "windows", # OS emulation: windows, macos (default), linux, android, ios
|
|
80
|
+
header_order: [ # wire order of headers (names only, case-sensitive)
|
|
81
|
+
"host", # listed headers appear first in the given order, remaining
|
|
82
|
+
"user-agent", # emulation headers follow.
|
|
83
|
+
"accept",
|
|
84
|
+
],
|
|
79
85
|
headers: { # default headers for all requests
|
|
80
86
|
"Accept" => "application/json"
|
|
81
|
-
}
|
|
87
|
+
},
|
|
88
|
+
referer: true, # auto-set Referer header on redirects (default: true)
|
|
89
|
+
pool_max_idle_per_host: 10, # max idle connections per host
|
|
90
|
+
pool_max_size: 100, # max total connections in the pool
|
|
91
|
+
tcp_nodelay: true, # disable Nagle algorithm (default: true)
|
|
92
|
+
tcp_keepalive: 15, # SO_KEEPALIVE interval in seconds (default: 15)
|
|
93
|
+
local_address: "1.2.3.4", # bind outgoing connections to this source IP
|
|
94
|
+
tls_sni: true, # send SNI in TLS handshake (default: true)
|
|
95
|
+
min_tls_version: "tls1.2", # minimum TLS version: tls1.0, tls1.1, tls1.2, tls1.3
|
|
96
|
+
max_tls_version: "tls1.3", # maximum TLS version
|
|
82
97
|
)
|
|
83
98
|
|
|
84
99
|
resp = client.get("https://api.example.com/data")
|
|
@@ -99,6 +114,19 @@ All methods are available on both `Wreq` (module-level) and `Wreq::Client` (inst
|
|
|
99
114
|
| `head(url, **opts)` | HEAD request |
|
|
100
115
|
| `options(url, **opts)` | OPTIONS request |
|
|
101
116
|
|
|
117
|
+
### Cancelling Requests
|
|
118
|
+
|
|
119
|
+
Call `cancel` on a client to abort all in-flight requests and close their underlying connections immediately:
|
|
120
|
+
|
|
121
|
+
```ruby
|
|
122
|
+
client = Wreq::Client.new
|
|
123
|
+
|
|
124
|
+
# From another thread:
|
|
125
|
+
t = Thread.new { client.get("https://slow.example.com/big-download") }
|
|
126
|
+
sleep 1
|
|
127
|
+
client.cancel # all in-flight requests return with "request interrupted" error
|
|
128
|
+
```
|
|
129
|
+
|
|
102
130
|
### Per-Request Options
|
|
103
131
|
|
|
104
132
|
Pass an options hash as the second argument to any HTTP method:
|
|
@@ -116,6 +144,7 @@ Pass an options hash as the second argument to any HTTP method:
|
|
|
116
144
|
| `basic` | Array | `[username, password]` for Basic auth |
|
|
117
145
|
| `proxy` | String | Per-request proxy URL |
|
|
118
146
|
| `emulation` | String/Boolean | Per-request emulation override |
|
|
147
|
+
| `emulation_os` | String | OS emulation: `windows`, `macos`, `linux`, `android`, `ios` |
|
|
119
148
|
|
|
120
149
|
## Browser Emulation
|
|
121
150
|
|
|
@@ -126,17 +155,21 @@ resp = Wreq.get("https://tls.peet.ws/api/all")
|
|
|
126
155
|
|
|
127
156
|
# Explicit browser emulation
|
|
128
157
|
client = Wreq::Client.new(emulation: "firefox_146")
|
|
129
|
-
client = Wreq::Client.new(emulation: "
|
|
158
|
+
client = Wreq::Client.new(emulation: "safari_18.5")
|
|
130
159
|
client = Wreq::Client.new(emulation: "edge_142")
|
|
131
160
|
|
|
132
161
|
# Disable emulation entirely
|
|
133
162
|
client = Wreq::Client.new(emulation: false)
|
|
134
163
|
|
|
164
|
+
# Emulate a specific OS (default is macOS)
|
|
165
|
+
client = Wreq::Client.new(emulation: "chrome_145", emulation_os: "windows")
|
|
166
|
+
client = Wreq::Client.new(emulation: "chrome_145", emulation_os: "linux")
|
|
167
|
+
|
|
135
168
|
# Emulation + custom user-agent (user_agent overrides emulation's UA)
|
|
136
169
|
client = Wreq::Client.new(emulation: "chrome_143", user_agent: "MyBot/1.0")
|
|
137
170
|
|
|
138
171
|
# Per-request emulation override
|
|
139
|
-
resp = client.get("https://example.com", emulation: "
|
|
172
|
+
resp = client.get("https://example.com", emulation: "safari_26.2")
|
|
140
173
|
```
|
|
141
174
|
|
|
142
175
|
### Supported Browsers
|
data/exe/wreq
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "optparse"
|
|
5
|
+
require "json"
|
|
6
|
+
require "wreq-rb"
|
|
7
|
+
|
|
8
|
+
cfg = {
|
|
9
|
+
method: nil,
|
|
10
|
+
headers: {},
|
|
11
|
+
include: false,
|
|
12
|
+
silent: false,
|
|
13
|
+
redirect: 10,
|
|
14
|
+
verify_cert: true,
|
|
15
|
+
verify_host: true,
|
|
16
|
+
pretty: false,
|
|
17
|
+
}
|
|
18
|
+
req = {}
|
|
19
|
+
|
|
20
|
+
parser = OptionParser.new do |o|
|
|
21
|
+
o.banner = "Usage: wreq [METHOD] URL [options]"
|
|
22
|
+
|
|
23
|
+
o.separator ""
|
|
24
|
+
o.separator "Request:"
|
|
25
|
+
|
|
26
|
+
o.on("-X", "--request METHOD", "HTTP method (default: GET)") { |m| cfg[:method] = m.upcase }
|
|
27
|
+
|
|
28
|
+
o.on("-H", "--header LINE", 'Request header, e.g. "Accept: application/json" (repeatable)') do |h|
|
|
29
|
+
name, _, value = h.partition(":")
|
|
30
|
+
cfg[:headers][name.strip] = value.strip
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
o.on("-d", "--data DATA",
|
|
34
|
+
"Request body. Prefix with @ to read from file, e.g. @body.json") do |d|
|
|
35
|
+
cfg[:method] ||= "POST"
|
|
36
|
+
if d.start_with?("@")
|
|
37
|
+
path = d[1..]
|
|
38
|
+
abort "wreq: cannot read '#{path}': file not found" unless File.exist?(path)
|
|
39
|
+
req[:body] = File.binread(path)
|
|
40
|
+
else
|
|
41
|
+
req[:body] = d
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
o.on("--json DATA",
|
|
46
|
+
"JSON body (sets Content-Type: application/json). Prefix with @ for file.") do |d|
|
|
47
|
+
cfg[:method] ||= "POST"
|
|
48
|
+
raw = d.start_with?("@") ? File.read(d[1..]) : d
|
|
49
|
+
begin
|
|
50
|
+
req[:json] = JSON.parse(raw)
|
|
51
|
+
rescue JSON::ParserError => e
|
|
52
|
+
abort "wreq: invalid JSON: #{e.message}"
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
o.on("--form PAIR", "Form field as key=value (repeatable)") do |f|
|
|
57
|
+
cfg[:method] ||= "POST"
|
|
58
|
+
key, _, val = f.partition("=")
|
|
59
|
+
(req[:form] ||= {})[key] = val
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
o.on("-q", "--query PAIR", "Query parameter as key=value (repeatable)") do |q|
|
|
63
|
+
key, _, val = q.partition("=")
|
|
64
|
+
(req[:query] ||= {})[key] = val
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
o.separator ""
|
|
68
|
+
o.separator "Auth:"
|
|
69
|
+
|
|
70
|
+
o.on("-u", "--user USER[:PASS]", "Basic auth credentials") do |u|
|
|
71
|
+
user, _, pass = u.partition(":")
|
|
72
|
+
req[:basic] = [user, pass]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
o.on("--bearer TOKEN", "Bearer token auth") { |t| req[:bearer] = t }
|
|
76
|
+
|
|
77
|
+
o.separator ""
|
|
78
|
+
o.separator "Output:"
|
|
79
|
+
|
|
80
|
+
o.on("-i", "--include", "Include response headers in stdout before the body") do
|
|
81
|
+
cfg[:include] = true
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
o.on("-s", "--silent", "Suppress body output") { cfg[:silent] = true }
|
|
85
|
+
|
|
86
|
+
o.on("-o", "--output FILE", "Write body to FILE") { |f| cfg[:output] = f }
|
|
87
|
+
|
|
88
|
+
o.on("--pretty", "Pretty-print JSON response bodies") { cfg[:pretty] = true }
|
|
89
|
+
|
|
90
|
+
o.separator ""
|
|
91
|
+
o.separator "Connection:"
|
|
92
|
+
|
|
93
|
+
o.on("-L", "--location", "Follow redirects (default: on, max 10)") { cfg[:redirect] = 10 }
|
|
94
|
+
o.on("--no-location", "Do not follow redirects") { cfg[:redirect] = false }
|
|
95
|
+
o.on("--max-redirects N", Integer, "Max redirects (default: 10)") { |n| cfg[:redirect] = n }
|
|
96
|
+
|
|
97
|
+
o.on("-k", "--insecure", "Skip TLS certificate verification") do
|
|
98
|
+
cfg[:verify_cert] = false
|
|
99
|
+
cfg[:verify_host] = false
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
o.on("-x", "--proxy URL", "Proxy URL (e.g. http://host:port)") { |p| cfg[:proxy] = p }
|
|
103
|
+
|
|
104
|
+
o.on("--local-address IP", "Bind outgoing connections to this source IP address") do |ip|
|
|
105
|
+
cfg[:local_address] = ip
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
o.on("--timeout SECS", Float, "Total timeout in seconds") { |t| cfg[:timeout] = t }
|
|
109
|
+
o.on("--connect-timeout SECS", Float, "Connection timeout in seconds") { |t| cfg[:connect_timeout] = t }
|
|
110
|
+
|
|
111
|
+
o.on("--http1", "Force HTTP/1.1") { cfg[:http1_only] = true }
|
|
112
|
+
o.on("--http2", "Force HTTP/2") { cfg[:http2_only] = true }
|
|
113
|
+
|
|
114
|
+
o.separator ""
|
|
115
|
+
o.separator "Emulation:"
|
|
116
|
+
|
|
117
|
+
o.on("--emulation PROFILE",
|
|
118
|
+
"Browser emulation profile (e.g. chrome_145, firefox_147, safari_26.2)") do |e|
|
|
119
|
+
cfg[:emulation] = e
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
o.on("--emulation-os OS",
|
|
123
|
+
"Emulation OS override: windows, macos, linux, android, ios") do |os|
|
|
124
|
+
cfg[:emulation_os] = os
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
o.on("--no-emulation", "Disable browser emulation entirely") { cfg[:emulation] = false }
|
|
128
|
+
|
|
129
|
+
o.on("-A", "--user-agent UA", "Override User-Agent header") { |ua| cfg[:user_agent] = ua }
|
|
130
|
+
|
|
131
|
+
o.on("--cookie-store", "Enable persistent cookie jar for this request") { cfg[:cookie_store] = true }
|
|
132
|
+
|
|
133
|
+
o.separator ""
|
|
134
|
+
|
|
135
|
+
o.on("-V", "--version", "Print version and exit") { puts "wreq #{Wreq::VERSION}"; exit 0 }
|
|
136
|
+
o.on_tail("-h", "--help", "Show this help") { puts o; exit 0 }
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
args = ARGV.dup
|
|
140
|
+
|
|
141
|
+
# Allow bare method as first positional arg: `wreq POST https://...`
|
|
142
|
+
if args.first =~ /\A[A-Z]+\z/
|
|
143
|
+
cfg[:method] = args.shift
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
begin
|
|
147
|
+
parser.parse!(args)
|
|
148
|
+
rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
|
|
149
|
+
abort "wreq: #{e.message}\nRun 'wreq --help' for usage."
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
url = args.shift
|
|
153
|
+
unless url
|
|
154
|
+
$stderr.puts parser
|
|
155
|
+
exit 1
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
cfg[:method] ||= "GET"
|
|
159
|
+
|
|
160
|
+
# Build Wreq::Client options (connection/emulation settings only)
|
|
161
|
+
CLIENT_KEYS = %i[emulation emulation_os user_agent proxy local_address timeout connect_timeout
|
|
162
|
+
http1_only http2_only cookie_store https_only].freeze
|
|
163
|
+
client_opts = cfg.slice(*CLIENT_KEYS).reject { |_, v| v.nil? }
|
|
164
|
+
client_opts[:redirect] = cfg[:redirect]
|
|
165
|
+
client_opts[:verify_cert] = cfg[:verify_cert]
|
|
166
|
+
client_opts[:verify_host] = cfg[:verify_host]
|
|
167
|
+
|
|
168
|
+
client = Wreq::Client.new(**client_opts)
|
|
169
|
+
|
|
170
|
+
# Per-request headers
|
|
171
|
+
req[:headers] = cfg[:headers] unless cfg[:headers].empty?
|
|
172
|
+
|
|
173
|
+
# Execute request
|
|
174
|
+
begin
|
|
175
|
+
response = client.public_send(cfg[:method].downcase, url, **req)
|
|
176
|
+
rescue NoMethodError
|
|
177
|
+
abort "wreq: unsupported HTTP method '#{cfg[:method]}'"
|
|
178
|
+
rescue => e
|
|
179
|
+
abort "wreq: #{e.message}"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# -i: print response status + headers to stdout before body
|
|
183
|
+
if cfg[:include]
|
|
184
|
+
puts "#{response.version} #{response.status}"
|
|
185
|
+
response.headers.each do |k, values|
|
|
186
|
+
Array(values).each { |v| puts "#{k}: #{v}" }
|
|
187
|
+
end
|
|
188
|
+
puts ""
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Output body
|
|
192
|
+
unless cfg[:silent] || cfg[:method] == "HEAD"
|
|
193
|
+
if cfg[:output]
|
|
194
|
+
bytes = response.body_bytes.pack("C*")
|
|
195
|
+
File.binwrite(cfg[:output], bytes)
|
|
196
|
+
warn "Saved #{bytes.bytesize} bytes to #{cfg[:output]}"
|
|
197
|
+
else
|
|
198
|
+
body = response.body
|
|
199
|
+
if cfg[:pretty] && !body.nil? && !body.empty?
|
|
200
|
+
body = begin
|
|
201
|
+
JSON.pretty_generate(JSON.parse(body))
|
|
202
|
+
rescue JSON::ParserError
|
|
203
|
+
body
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
print body unless body.nil?
|
|
207
|
+
$stdout.puts if body && !body.empty? && !body.end_with?("\n")
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
exit response.success? ? 0 : 1
|
data/ext/wreq_rb/Cargo.toml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "wreq_rb"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.5.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
publish = false
|
|
6
6
|
|
|
@@ -24,7 +24,7 @@ wreq = { path = "../../vendor/wreq", features = [
|
|
|
24
24
|
"query",
|
|
25
25
|
"form",
|
|
26
26
|
] }
|
|
27
|
-
wreq-util = { version = "=3.0.0-rc.10", features = ["emulation", "emulation-serde"] }
|
|
27
|
+
wreq-util = { version = "=3.0.0-rc.10", features = ["emulation", "emulation-serde", "emulation-compression"] }
|
|
28
28
|
tokio = { version = "1", features = ["full"] }
|
|
29
29
|
tokio-util = "0.7"
|
|
30
30
|
serde_json = "1.0"
|
data/ext/wreq_rb/src/client.rs
CHANGED
|
@@ -10,8 +10,10 @@ use magnus::{
|
|
|
10
10
|
};
|
|
11
11
|
use tokio::runtime::Runtime;
|
|
12
12
|
use tokio_util::sync::CancellationToken;
|
|
13
|
-
use
|
|
14
|
-
use
|
|
13
|
+
use std::net::IpAddr;
|
|
14
|
+
use wreq::header::{HeaderMap, HeaderName, HeaderValue, OrigHeaderMap};
|
|
15
|
+
use wreq::tls::TlsVersion;
|
|
16
|
+
use wreq_util::{Emulation as BrowserEmulation, EmulationOS, EmulationOption};
|
|
15
17
|
|
|
16
18
|
use crate::error::{generic_error, to_magnus_error};
|
|
17
19
|
use crate::response::Response;
|
|
@@ -149,6 +151,28 @@ fn parse_emulation(name: &str) -> Result<BrowserEmulation, magnus::Error> {
|
|
|
149
151
|
.map_err(|_| generic_error(format!("unknown emulation: '{}'. Use names like 'chrome_145', 'firefox_147', 'safari_18.5', etc.", name)))
|
|
150
152
|
}
|
|
151
153
|
|
|
154
|
+
/// Parse a Ruby string like "windows" into an EmulationOS variant.
|
|
155
|
+
fn parse_emulation_os(name: &str) -> Result<EmulationOS, magnus::Error> {
|
|
156
|
+
let json_val = serde_json::Value::String(name.to_string());
|
|
157
|
+
serde_json::from_value::<EmulationOS>(json_val)
|
|
158
|
+
.map_err(|_| generic_error("unknown emulation_os. Use: 'windows', 'macos', 'linux', 'android', 'ios'"))
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/// Build an EmulationOption from an Emulation and an optional OS from the opts hash.
|
|
162
|
+
fn build_emulation_option(
|
|
163
|
+
emu: BrowserEmulation,
|
|
164
|
+
opts: &RHash,
|
|
165
|
+
) -> Result<EmulationOption, magnus::Error> {
|
|
166
|
+
let os = match hash_get_string(opts, "emulation_os")? {
|
|
167
|
+
Some(os_name) => parse_emulation_os(&os_name)?,
|
|
168
|
+
None => EmulationOS::default(),
|
|
169
|
+
};
|
|
170
|
+
Ok(EmulationOption::builder()
|
|
171
|
+
.emulation(emu)
|
|
172
|
+
.emulation_os(os)
|
|
173
|
+
.build())
|
|
174
|
+
}
|
|
175
|
+
|
|
152
176
|
// --------------------------------------------------------------------------
|
|
153
177
|
// Ruby Client
|
|
154
178
|
// --------------------------------------------------------------------------
|
|
@@ -156,6 +180,7 @@ fn parse_emulation(name: &str) -> Result<BrowserEmulation, magnus::Error> {
|
|
|
156
180
|
#[magnus::wrap(class = "Wreq::Client", free_immediately)]
|
|
157
181
|
struct Client {
|
|
158
182
|
inner: wreq::Client,
|
|
183
|
+
cancel_token: std::sync::Mutex<CancellationToken>,
|
|
159
184
|
}
|
|
160
185
|
|
|
161
186
|
impl Client {
|
|
@@ -170,19 +195,28 @@ impl Client {
|
|
|
170
195
|
let mut builder = wreq::Client::builder();
|
|
171
196
|
|
|
172
197
|
if let Some(opts) = opts {
|
|
198
|
+
// Apply header_order BEFORE emulation so the user's ordering takes precedence
|
|
199
|
+
if let Some(ary) = hash_get_array(&opts, "header_order")? {
|
|
200
|
+
let orig = array_to_orig_header_map(ary)?;
|
|
201
|
+
builder = builder.orig_headers(orig);
|
|
202
|
+
}
|
|
203
|
+
|
|
173
204
|
if let Some(val) = hash_get_value(&opts, "emulation")? {
|
|
174
205
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
175
206
|
if val.is_kind_of(ruby.class_false_class()) {
|
|
176
207
|
// emulation: false — skip emulation entirely
|
|
177
208
|
} else if val.is_kind_of(ruby.class_true_class()) {
|
|
178
|
-
|
|
209
|
+
let opt = build_emulation_option(DEFAULT_EMULATION, &opts)?;
|
|
210
|
+
builder = builder.emulation(opt);
|
|
179
211
|
} else {
|
|
180
212
|
let name: String = TryConvert::try_convert(val)?;
|
|
181
213
|
let emu = parse_emulation(&name)?;
|
|
182
|
-
|
|
214
|
+
let opt = build_emulation_option(emu, &opts)?;
|
|
215
|
+
builder = builder.emulation(opt);
|
|
183
216
|
}
|
|
184
217
|
} else {
|
|
185
|
-
|
|
218
|
+
let opt = build_emulation_option(DEFAULT_EMULATION, &opts)?;
|
|
219
|
+
builder = builder.emulation(opt);
|
|
186
220
|
}
|
|
187
221
|
|
|
188
222
|
if let Some(ua) = hash_get_string(&opts, "user_agent")? {
|
|
@@ -268,12 +302,50 @@ impl Client {
|
|
|
268
302
|
if let Some(v) = hash_get_bool(&opts, "zstd")? {
|
|
269
303
|
builder = builder.zstd(v);
|
|
270
304
|
}
|
|
305
|
+
|
|
306
|
+
if let Some(v) = hash_get_bool(&opts, "referer")? {
|
|
307
|
+
builder = builder.referer(v);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
if let Some(n) = hash_get_usize(&opts, "pool_max_idle_per_host")? {
|
|
311
|
+
builder = builder.pool_max_idle_per_host(n);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
if let Some(n) = hash_get_u32(&opts, "pool_max_size")? {
|
|
315
|
+
builder = builder.pool_max_size(n);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
if let Some(v) = hash_get_bool(&opts, "tcp_nodelay")? {
|
|
319
|
+
builder = builder.tcp_nodelay(v);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
if let Some(t) = hash_get_float(&opts, "tcp_keepalive")? {
|
|
323
|
+
builder = builder.tcp_keepalive(Duration::from_secs_f64(t));
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
if let Some(addr_str) = hash_get_string(&opts, "local_address")? {
|
|
327
|
+
let addr: IpAddr = addr_str.parse()
|
|
328
|
+
.map_err(|_| generic_error(format!("invalid IP address: '{}'", addr_str)))?;
|
|
329
|
+
builder = builder.local_address(addr);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
if let Some(v) = hash_get_bool(&opts, "tls_sni")? {
|
|
333
|
+
builder = builder.tls_sni(v);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if let Some(s) = hash_get_string(&opts, "min_tls_version")? {
|
|
337
|
+
builder = builder.min_tls_version(parse_tls_version(&s)?);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if let Some(s) = hash_get_string(&opts, "max_tls_version")? {
|
|
341
|
+
builder = builder.max_tls_version(parse_tls_version(&s)?);
|
|
342
|
+
}
|
|
271
343
|
} else {
|
|
272
344
|
builder = builder.emulation(DEFAULT_EMULATION);
|
|
273
345
|
}
|
|
274
346
|
|
|
275
347
|
let client = builder.build().map_err(to_magnus_error)?;
|
|
276
|
-
Ok(Client { inner: client })
|
|
348
|
+
Ok(Client { inner: client, cancel_token: std::sync::Mutex::new(CancellationToken::new()) })
|
|
277
349
|
}
|
|
278
350
|
|
|
279
351
|
/// client.get(url) or client.get(url, opts)
|
|
@@ -305,6 +377,19 @@ impl Client {
|
|
|
305
377
|
self.execute_method("OPTIONS", args)
|
|
306
378
|
}
|
|
307
379
|
|
|
380
|
+
fn cancel(&self) {
|
|
381
|
+
// Replace the cancel token first so new requests use a fresh token,
|
|
382
|
+
// then cancel the old one to unblock all current in-flight select!s.
|
|
383
|
+
let old_token = {
|
|
384
|
+
let mut guard = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner());
|
|
385
|
+
let old = guard.clone();
|
|
386
|
+
*guard = CancellationToken::new();
|
|
387
|
+
old
|
|
388
|
+
};
|
|
389
|
+
old_token.cancel();
|
|
390
|
+
self.inner.cancel_connections();
|
|
391
|
+
}
|
|
392
|
+
|
|
308
393
|
fn execute_method(&self, method_str: &str, args: &[Value]) -> Result<Response, magnus::Error> {
|
|
309
394
|
let url: String = if args.is_empty() {
|
|
310
395
|
return Err(generic_error("url is required"));
|
|
@@ -328,13 +413,16 @@ impl Client {
|
|
|
328
413
|
req = apply_request_options(req, &opts)?;
|
|
329
414
|
}
|
|
330
415
|
|
|
416
|
+
let client_token = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
|
417
|
+
|
|
331
418
|
// Release the GVL so other Ruby threads can run during I/O.
|
|
332
419
|
let outcome: RequestOutcome = unsafe {
|
|
333
|
-
without_gvl(|
|
|
420
|
+
without_gvl(|thread_token| {
|
|
334
421
|
runtime().block_on(async {
|
|
335
422
|
tokio::select! {
|
|
336
423
|
biased;
|
|
337
|
-
_ =
|
|
424
|
+
_ = thread_token.cancelled() => RequestOutcome::Interrupted,
|
|
425
|
+
_ = client_token.cancelled() => RequestOutcome::Interrupted,
|
|
338
426
|
res = execute_request(req) => match res {
|
|
339
427
|
Ok(data) => RequestOutcome::Ok(data),
|
|
340
428
|
Err(e) => RequestOutcome::Err(e),
|
|
@@ -416,11 +504,13 @@ fn apply_request_options(
|
|
|
416
504
|
if val.is_kind_of(ruby.class_false_class()) {
|
|
417
505
|
// emulation: false — no per-request emulation override
|
|
418
506
|
} else if val.is_kind_of(ruby.class_true_class()) {
|
|
419
|
-
|
|
507
|
+
let opt = build_emulation_option(DEFAULT_EMULATION, opts)?;
|
|
508
|
+
req = req.emulation(opt);
|
|
420
509
|
} else {
|
|
421
510
|
let name: String = TryConvert::try_convert(val)?;
|
|
422
511
|
let emu = parse_emulation(&name)?;
|
|
423
|
-
|
|
512
|
+
let opt = build_emulation_option(emu, opts)?;
|
|
513
|
+
req = req.emulation(opt);
|
|
424
514
|
}
|
|
425
515
|
}
|
|
426
516
|
|
|
@@ -502,6 +592,32 @@ fn hash_get_bool(hash: &RHash, key: &str) -> Result<Option<bool>, magnus::Error>
|
|
|
502
592
|
}
|
|
503
593
|
}
|
|
504
594
|
|
|
595
|
+
fn hash_get_usize(hash: &RHash, key: &str) -> Result<Option<usize>, magnus::Error> {
|
|
596
|
+
match hash_get_value(hash, key)? {
|
|
597
|
+
Some(v) => Ok(Some(TryConvert::try_convert(v)?)),
|
|
598
|
+
None => Ok(None),
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
fn hash_get_u32(hash: &RHash, key: &str) -> Result<Option<u32>, magnus::Error> {
|
|
603
|
+
match hash_get_value(hash, key)? {
|
|
604
|
+
Some(v) => Ok(Some(TryConvert::try_convert(v)?)),
|
|
605
|
+
None => Ok(None),
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
fn parse_tls_version(s: &str) -> Result<TlsVersion, magnus::Error> {
|
|
610
|
+
match s {
|
|
611
|
+
"tls1.0" | "tls_1_0" | "1.0" => Ok(TlsVersion::TLS_1_0),
|
|
612
|
+
"tls1.1" | "tls_1_1" | "1.1" => Ok(TlsVersion::TLS_1_1),
|
|
613
|
+
"tls1.2" | "tls_1_2" | "1.2" => Ok(TlsVersion::TLS_1_2),
|
|
614
|
+
"tls1.3" | "tls_1_3" | "1.3" => Ok(TlsVersion::TLS_1_3),
|
|
615
|
+
_ => Err(generic_error(format!(
|
|
616
|
+
"unknown TLS version '{}'. Use: 'tls1.2', 'tls1.3'", s
|
|
617
|
+
))),
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
505
621
|
fn hash_get_hash(hash: &RHash, key: &str) -> Result<Option<RHash>, magnus::Error> {
|
|
506
622
|
match hash_get_value(hash, key)? {
|
|
507
623
|
Some(v) => Ok(Some(RHash::try_convert(v)?)),
|
|
@@ -509,6 +625,27 @@ fn hash_get_hash(hash: &RHash, key: &str) -> Result<Option<RHash>, magnus::Error
|
|
|
509
625
|
}
|
|
510
626
|
}
|
|
511
627
|
|
|
628
|
+
fn hash_get_array(hash: &RHash, key: &str) -> Result<Option<RArray>, magnus::Error> {
|
|
629
|
+
match hash_get_value(hash, key)? {
|
|
630
|
+
Some(v) => Ok(Some(RArray::try_convert(v)?)),
|
|
631
|
+
None => Ok(None),
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
fn array_to_orig_header_map(ary: RArray) -> Result<OrigHeaderMap, magnus::Error> {
|
|
636
|
+
let mut orig = OrigHeaderMap::with_capacity(ary.len());
|
|
637
|
+
for elem in ary.into_iter() {
|
|
638
|
+
let ruby = unsafe { Ruby::get_unchecked() };
|
|
639
|
+
let name_str: String = if elem.is_kind_of(ruby.class_symbol()) {
|
|
640
|
+
elem.funcall("to_s", ())?
|
|
641
|
+
} else {
|
|
642
|
+
TryConvert::try_convert(elem)?
|
|
643
|
+
};
|
|
644
|
+
orig.insert(name_str);
|
|
645
|
+
}
|
|
646
|
+
Ok(orig)
|
|
647
|
+
}
|
|
648
|
+
|
|
512
649
|
fn hash_to_header_map(hash: &RHash) -> Result<HeaderMap, magnus::Error> {
|
|
513
650
|
let mut hmap = HeaderMap::new();
|
|
514
651
|
hash.foreach(|k: Value, v: Value| {
|
|
@@ -562,6 +699,7 @@ pub fn init(_ruby: &magnus::Ruby, module: &magnus::RModule) -> Result<(), magnus
|
|
|
562
699
|
client_class.define_method("delete", method!(Client::delete, -1))?;
|
|
563
700
|
client_class.define_method("head", method!(Client::head, -1))?;
|
|
564
701
|
client_class.define_method("options", method!(Client::options, -1))?;
|
|
702
|
+
client_class.define_method("cancel", method!(Client::cancel, 0))?;
|
|
565
703
|
|
|
566
704
|
module.define_module_function("get", function!(wreq_get, -1))?;
|
|
567
705
|
module.define_module_function("post", function!(wreq_post, -1))?;
|
data/lib/wreq-rb/version.rb
CHANGED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
diff --git a/src/client/http.rs b/src/client/http.rs
|
|
2
|
+
index 462c77d0..6231e700 100644
|
|
3
|
+
--- a/src/client/http.rs
|
|
4
|
+
+++ b/src/client/http.rs
|
|
5
|
+
@@ -171,6 +171,7 @@ type ClientRef = Either<ClientService, BoxedClientService>;
|
|
6
|
+
#[derive(Clone)]
|
|
7
|
+
pub struct Client {
|
|
8
|
+
inner: Arc<ClientRef>,
|
|
9
|
+
+ cancel_connections: Arc<dyn Fn() + Send + Sync>,
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/// A [`ClientBuilder`] can be used to create a [`Client`] with custom configuration.
|
|
13
|
+
@@ -439,6 +440,14 @@ impl Client {
|
|
14
|
+
let fut = Oneshot::new(self.inner.as_ref().clone(), req);
|
|
15
|
+
Pending::request(uri, fut)
|
|
16
|
+
}
|
|
17
|
+
+
|
|
18
|
+
+ /// Cancel all background connection tasks immediately.
|
|
19
|
+
+ ///
|
|
20
|
+
+ /// This force-drops every tracked H1/H2 connection task, causing the
|
|
21
|
+
+ /// underlying TCP socket to close abruptly.
|
|
22
|
+
+ pub fn cancel_connections(&self) {
|
|
23
|
+
+ (self.cancel_connections)()
|
|
24
|
+
+ }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
impl tower::Service<Request> for Client {
|
|
28
|
+
@@ -495,7 +504,7 @@ impl ClientBuilder {
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Create base client service
|
|
32
|
+
- let service = {
|
|
33
|
+
+ let (service, cancel_connections) = {
|
|
34
|
+
let (tls_options, http1_options, http2_options) = config.transport_options.into();
|
|
35
|
+
|
|
36
|
+
let resolver = {
|
|
37
|
+
@@ -555,7 +564,7 @@ impl ClientBuilder {
|
|
38
|
+
.build(config.connector_layers)?;
|
|
39
|
+
|
|
40
|
+
// Build client
|
|
41
|
+
- HttpClient::builder(TokioExecutor::new())
|
|
42
|
+
+ let http_client = HttpClient::builder(TokioExecutor::new())
|
|
43
|
+
.http1_options(http1_options)
|
|
44
|
+
.http2_options(http2_options)
|
|
45
|
+
.http2_only(matches!(config.http_version_pref, HttpVersionPref::Http2))
|
|
46
|
+
@@ -564,8 +573,10 @@ impl ClientBuilder {
|
|
47
|
+
.pool_idle_timeout(config.pool_idle_timeout)
|
|
48
|
+
.pool_max_idle_per_host(config.pool_max_idle_per_host)
|
|
49
|
+
.pool_max_size(config.pool_max_size)
|
|
50
|
+
- .build(connector)
|
|
51
|
+
- .map_err(Into::into as _)
|
|
52
|
+
+ .build(connector);
|
|
53
|
+
+ // Capture abort fn before tower layers consume the client
|
|
54
|
+
+ let cancel_connections = http_client.cancel_connections_fn();
|
|
55
|
+
+ (http_client.map_err(Into::into as _), cancel_connections)
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
// Configured client service with layers
|
|
59
|
+
@@ -633,6 +644,7 @@ impl ClientBuilder {
|
|
60
|
+
|
|
61
|
+
Ok(Client {
|
|
62
|
+
inner: Arc::new(client),
|
|
63
|
+
+ cancel_connections,
|
|
64
|
+
})
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
diff --git a/src/client/http/client.rs b/src/client/http/client.rs
|
|
68
|
+
index a29c52dd..7f6ce68f 100644
|
|
69
|
+
--- a/src/client/http/client.rs
|
|
70
|
+
+++ b/src/client/http/client.rs
|
|
71
|
+
@@ -10,7 +10,7 @@ use std::{
|
|
72
|
+
future::Future,
|
|
73
|
+
num::NonZeroU32,
|
|
74
|
+
pin::Pin,
|
|
75
|
+
- sync::Arc,
|
|
76
|
+
+ sync::{Arc, Mutex},
|
|
77
|
+
task::{self, Poll},
|
|
78
|
+
time::Duration,
|
|
79
|
+
};
|
|
80
|
+
@@ -119,6 +119,7 @@ pub struct HttpClient<C, B> {
|
|
81
|
+
h1_builder: conn::http1::Builder,
|
|
82
|
+
h2_builder: conn::http2::Builder<Exec>,
|
|
83
|
+
pool: pool::Pool<PoolClient<B>, ConnectIdentity>,
|
|
84
|
+
+ conn_abort_handles: Arc<Mutex<Vec<tokio::task::AbortHandle>>>,
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
#[derive(Clone, Copy)]
|
|
88
|
+
@@ -452,8 +453,8 @@ where
|
|
89
|
+
+ Send
|
|
90
|
+
+ Unpin
|
|
91
|
+
+ 'static {
|
|
92
|
+
- let executor = self.exec.clone();
|
|
93
|
+
let pool = self.pool.clone();
|
|
94
|
+
+ let conn_abort_handles = self.conn_abort_handles.clone();
|
|
95
|
+
|
|
96
|
+
let h1_builder = self.h1_builder.clone();
|
|
97
|
+
let h2_builder = self.h2_builder.clone();
|
|
98
|
+
@@ -513,10 +514,17 @@ where
|
|
99
|
+
trace!(
|
|
100
|
+
"http2 handshake complete, spawning background dispatcher task"
|
|
101
|
+
);
|
|
102
|
+
- executor.execute(
|
|
103
|
+
- conn.map_err(|_e| debug!("client connection error: {}", _e))
|
|
104
|
+
- .map(|_| ()),
|
|
105
|
+
- );
|
|
106
|
+
+ let h2_conn_future = conn
|
|
107
|
+
+ .map_err(|_e| debug!("client connection error: {}", _e))
|
|
108
|
+
+ .map(|_| ());
|
|
109
|
+
+ let abort_handle = tokio::spawn(h2_conn_future).abort_handle();
|
|
110
|
+
+ {
|
|
111
|
+
+ let mut handles = conn_abort_handles.lock().unwrap_or_else(|e| e.into_inner());
|
|
112
|
+
+ if handles.len() >= 64 {
|
|
113
|
+
+ handles.retain(|h| !h.is_finished());
|
|
114
|
+
+ }
|
|
115
|
+
+ handles.push(abort_handle);
|
|
116
|
+
+ }
|
|
117
|
+
|
|
118
|
+
// Wait for 'conn' to ready up before we
|
|
119
|
+
// declare this tx as usable
|
|
120
|
+
@@ -544,8 +552,7 @@ where
|
|
121
|
+
// Spawn the connection task in the background using the executor.
|
|
122
|
+
// The task manages the HTTP/1.1 connection, including upgrades (e.g., WebSocket).
|
|
123
|
+
// Errors are sent via err_tx to ensure they can be checked if the sender (tx) fails.
|
|
124
|
+
- executor.execute(
|
|
125
|
+
- conn.with_upgrades()
|
|
126
|
+
+ let h1_conn_future = conn.with_upgrades()
|
|
127
|
+
.map_err(|e| {
|
|
128
|
+
// Log the connection error at debug level for diagnostic purposes.
|
|
129
|
+
debug!("client connection error: {:?}", e);
|
|
130
|
+
@@ -555,8 +562,15 @@ where
|
|
131
|
+
// (e.g., if the receiver is dropped, which is handled later).
|
|
132
|
+
let _ = err_tx.send(e);
|
|
133
|
+
})
|
|
134
|
+
- .map(|_| ()),
|
|
135
|
+
- );
|
|
136
|
+
+ .map(|_| ());
|
|
137
|
+
+ let abort_handle = tokio::spawn(h1_conn_future).abort_handle();
|
|
138
|
+
+ {
|
|
139
|
+
+ let mut handles = conn_abort_handles.lock().unwrap_or_else(|e| e.into_inner());
|
|
140
|
+
+ if handles.len() >= 64 {
|
|
141
|
+
+ handles.retain(|h| !h.is_finished());
|
|
142
|
+
+ }
|
|
143
|
+
+ handles.push(abort_handle);
|
|
144
|
+
+ }
|
|
145
|
+
|
|
146
|
+
// Log that the client is waiting for the connection to be ready.
|
|
147
|
+
// Readiness indicates the sender (tx) can accept a request without blocking. More actions
|
|
148
|
+
@@ -682,10 +696,25 @@ impl<C: Clone, B> Clone for HttpClient<C, B> {
|
|
149
|
+
h2_builder: self.h2_builder.clone(),
|
|
150
|
+
connector: self.connector.clone(),
|
|
151
|
+
pool: self.pool.clone(),
|
|
152
|
+
+ conn_abort_handles: self.conn_abort_handles.clone(),
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
+impl<C, B> HttpClient<C, B> {
|
|
158
|
+
+ /// Returns a closure that cancels all tracked connection tasks when called.
|
|
159
|
+
+ /// Each call drains the handle list and aborts every stored task.
|
|
160
|
+
+ pub fn cancel_connections_fn(&self) -> Arc<dyn Fn() + Send + Sync + 'static> {
|
|
161
|
+
+ let handles = self.conn_abort_handles.clone();
|
|
162
|
+
+ Arc::new(move || {
|
|
163
|
+
+ let mut guard = handles.lock().unwrap_or_else(|e| e.into_inner());
|
|
164
|
+
+ for handle in guard.drain(..) {
|
|
165
|
+
+ handle.abort();
|
|
166
|
+
+ }
|
|
167
|
+
+ })
|
|
168
|
+
+ }
|
|
169
|
+
+}
|
|
170
|
+
+
|
|
171
|
+
/// A pooled HTTP connection that can send requests
|
|
172
|
+
struct PoolClient<B> {
|
|
173
|
+
conn_info: Connected,
|
|
174
|
+
@@ -998,6 +1027,7 @@ impl Builder {
|
|
175
|
+
h2_builder: self.h2_builder,
|
|
176
|
+
connector,
|
|
177
|
+
pool: pool::Pool::new(self.pool_config, exec, timer),
|
|
178
|
+
+ conn_abort_handles: Arc::new(Mutex::new(Vec::new())),
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
metadata
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: wreq-rb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yicheng Zhou
|
|
8
8
|
- Illia Zub
|
|
9
9
|
autorequire:
|
|
10
|
-
bindir:
|
|
10
|
+
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2026-03-
|
|
12
|
+
date: 2026-03-17 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: rb_sys
|
|
@@ -29,7 +29,8 @@ description: An ergonomic Ruby HTTP client powered by Rust's wreq library, featu
|
|
|
29
29
|
TLS fingerprint emulation (JA3/JA4), HTTP/2 support, cookie handling, proxy support,
|
|
30
30
|
and redirect policies.
|
|
31
31
|
email:
|
|
32
|
-
executables:
|
|
32
|
+
executables:
|
|
33
|
+
- wreq
|
|
33
34
|
extensions:
|
|
34
35
|
- ext/wreq_rb/extconf.rb
|
|
35
36
|
extra_rdoc_files: []
|
|
@@ -37,6 +38,7 @@ files:
|
|
|
37
38
|
- Cargo.lock
|
|
38
39
|
- Cargo.toml
|
|
39
40
|
- README.md
|
|
41
|
+
- exe/wreq
|
|
40
42
|
- ext/wreq_rb/Cargo.toml
|
|
41
43
|
- ext/wreq_rb/extconf.rb
|
|
42
44
|
- ext/wreq_rb/src/client.rs
|
|
@@ -46,6 +48,7 @@ files:
|
|
|
46
48
|
- lib/wreq-rb.rb
|
|
47
49
|
- lib/wreq-rb/version.rb
|
|
48
50
|
- patches/0001-add-transfer-size-tracking.patch
|
|
51
|
+
- patches/0002-add-cancel-connections.patch
|
|
49
52
|
- vendor/wreq/Cargo.toml
|
|
50
53
|
- vendor/wreq/LICENSE
|
|
51
54
|
- vendor/wreq/README.md
|