wreq 1.2.15 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +1 -1
- data/Cargo.toml +1 -1
- data/README.md +1 -1
- data/crates/wreq-util/CHANGELOG.md +3 -0
- data/crates/wreq-util/src/emulate/macros.rs +12 -17
- data/crates/wreq-util/src/emulate/profile/chrome/tls.rs +105 -2
- data/crates/wreq-util/src/emulate/profile/chrome.rs +34 -0
- data/crates/wreq-util/src/emulate/profile/firefox/header.rs +22 -0
- data/crates/wreq-util/src/emulate/profile/firefox.rs +6 -6
- data/crates/wreq-util/src/emulate.rs +6 -0
- data/crates/wreq-util/tests/emulate_chrome.rs +10 -0
- data/crates/wreq-util/tests/emulate_firefox.rs +58 -0
- data/lib/wreq_ruby/emulate.rb +13 -0
- data/src/emulate.rs +9 -3
- data/test/emulation_test.rb +77 -0
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a684244b1b3a988034a958ba6d5be4bd1c3233a958834eb103cabf8b57c7c7d2
|
|
4
|
+
data.tar.gz: 2246ef7c885c1d4c429b1a0630795b8fe24bd1923dd4205267dd727ff552bb90
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4a1e164422249f5cc1df7d1259879e640c4e822ef3e9e2d0d67c11128e7f90c023119ddb6d1610a098b1d2c43657f72cdf0f0780f29f80f81e58be6b2cf66f72
|
|
7
|
+
data.tar.gz: 8298504fd0e17008572c3d0a7ed0ee1ef949bbce9c88a73729fcece75ba87fb54c43ce38fa1ae961fcf297db8e0c2ca58b528155d14aac588ee23499255b860e
|
data/Cargo.lock
CHANGED
data/Cargo.toml
CHANGED
data/README.md
CHANGED
|
@@ -91,7 +91,7 @@ Most browser device models share identical TLS and HTTP/2 configurations, differ
|
|
|
91
91
|
|
|
92
92
|
| **Browser** | **Versions** |
|
|
93
93
|
| ----------- | ------------ |
|
|
94
|
-
| **Chrome** | `Chrome100`, `Chrome101`, `Chrome104`, `Chrome105`, `Chrome106`, `Chrome107`, `Chrome108`, `Chrome109`, `Chrome110`, `Chrome114`, `Chrome116`, `Chrome117`, `Chrome118`, `Chrome119`, `Chrome120`, `Chrome123`, `Chrome124`, `Chrome126`, `Chrome127`, `Chrome128`, `Chrome129`, `Chrome130`, `Chrome131`, `Chrome132`, `Chrome133`, `Chrome134`, `Chrome135`, `Chrome136`, `Chrome137`, `Chrome138`, `Chrome139`, `Chrome140`, `Chrome141`, `Chrome142`, `Chrome143`, `Chrome144`, `Chrome145`, `Chrome146`, `Chrome147`, `Chrome148`, `Chrome149`, `Chrome150`, `Chrome151`, `Chrome152`, `Chrome153` |
|
|
94
|
+
| **Chrome** | `Chrome100`, `Chrome101`, `Chrome104`, `Chrome105`, `Chrome106`, `Chrome107`, `Chrome108`, `Chrome109`, `Chrome110`, `Chrome114`, `Chrome116`, `Chrome117`, `Chrome118`, `Chrome119`, `Chrome120`, `Chrome123`, `Chrome124`, `Chrome126`, `Chrome127`, `Chrome128`, `Chrome129`, `Chrome130`, `Chrome131`, `Chrome132`, `Chrome133`, `Chrome134`, `Chrome135`, `Chrome136`, `Chrome137`, `Chrome138`, `Chrome139`, `Chrome140`, `Chrome141`, `Chrome142`, `Chrome143`, `Chrome144`, `Chrome145`, `Chrome146`, `Chrome147`, `Chrome148`, `Chrome149`, `Chrome150`, `Chrome151`, `Chrome152`, `Chrome153`, `Chrome154` |
|
|
95
95
|
| **Edge** | `Edge101`, `Edge122`, `Edge127`, `Edge131`, `Edge134`, `Edge135`, `Edge136`, `Edge137`, `Edge138`, `Edge139`, `Edge140`, `Edge141`, `Edge142`, `Edge143`, `Edge144`, `Edge145`, `Edge146`, `Edge147`, `Edge148` |
|
|
96
96
|
| **Firefox** | `Firefox109`, `Firefox117`, `Firefox128`, `Firefox133`, `Firefox135`, `FirefoxPrivate135`, `FirefoxAndroid135`, `Firefox136`, `FirefoxPrivate136`, `Firefox139`, `Firefox142`, `Firefox143`, `Firefox144`, `Firefox145`, `Firefox146`, `Firefox147`, `Firefox148`, `Firefox149`, `Firefox150`, `Firefox151`, `Firefox152` |
|
|
97
97
|
| **Safari** | `Safari15_3`, `Safari15_5`, `Safari15_6_1`, `Safari16`, `Safari16_5`, `Safari17_0`, `Safari17_2_1`, `Safari17_4_1`, `Safari17_5`, `Safari17_6`, `Safari18`, `Safari18_2`, `Safari18_3`, `Safari18_3_1`, `Safari18_5`, `Safari26`, `Safari26_1`, `Safari26_2`, `SafariIos16_5`, `SafariIos17_2`, `SafariIos17_4_1`, `SafariIos18_1_1`, `SafariIos26`, `SafariIos26_2`, `SafariIPad18`, `SafariIPad26`, `SafariIpad26_2` |
|
|
@@ -12,10 +12,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
12
12
|
- *(emulate)* Add Chrome 151 profile.
|
|
13
13
|
- *(emulate)* Add Chrome 152 profile.
|
|
14
14
|
- *(emulate)* Add Chrome 153 profile.
|
|
15
|
+
- *(emulate)* Add Chrome 154 profile.
|
|
15
16
|
- *(emulate)* Add Firefox 152 profile.
|
|
16
17
|
|
|
17
18
|
### Fixed
|
|
18
19
|
|
|
20
|
+
- Match Trust Anchor ID ordering by Chrome version: randomized per native configuration for 152/153, and sorted by raw ID bytes for 154.
|
|
21
|
+
|
|
19
22
|
- *(emulate)* Fix Firefox 150 and 151 cipher suite lists.
|
|
20
23
|
|
|
21
24
|
## [3.0.0-rc.14](https://github.com/0x676e67/wreq-util/compare/v3.0.0-rc.13...v3.0.0-rc.14) - 2026-07-04
|
|
@@ -151,32 +151,27 @@ macro_rules! header_firefox_sec_fetch {
|
|
|
151
151
|
|
|
152
152
|
macro_rules! header_firefox_accept {
|
|
153
153
|
($headers:expr) => {
|
|
154
|
-
$headers.
|
|
155
|
-
ACCEPT,
|
|
156
|
-
HeaderValue::from_static(
|
|
157
|
-
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
158
|
-
),
|
|
159
|
-
);
|
|
160
|
-
$headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
|
|
161
|
-
#[cfg(feature = "emulation-compression")]
|
|
162
|
-
$headers.insert(
|
|
163
|
-
ACCEPT_ENCODING,
|
|
164
|
-
HeaderValue::from_static("gzip, deflate, br"),
|
|
165
|
-
);
|
|
154
|
+
header_firefox_accept!(@build $headers, "en-US,en;q=0.5", "gzip, deflate, br");
|
|
166
155
|
};
|
|
167
156
|
(zstd, $headers:expr) => {
|
|
157
|
+
header_firefox_accept!(@build $headers, "en-US,en;q=0.5", "gzip, deflate, br, zstd");
|
|
158
|
+
};
|
|
159
|
+
// Firefox 147 changed the generated q-value from dividing 1.0 across the list to
|
|
160
|
+
// decrementing by 0.1 per entry, so a two-entry list became q=0.9 (Bugzilla 2000765,
|
|
161
|
+
// landed in 0e050ae5116d).
|
|
162
|
+
(zstd_q09, $headers:expr) => {
|
|
163
|
+
header_firefox_accept!(@build $headers, "en-US,en;q=0.9", "gzip, deflate, br, zstd");
|
|
164
|
+
};
|
|
165
|
+
(@build $headers:expr, $accept_language:expr, $accept_encoding:expr) => {
|
|
168
166
|
$headers.insert(
|
|
169
167
|
ACCEPT,
|
|
170
168
|
HeaderValue::from_static(
|
|
171
169
|
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
172
170
|
),
|
|
173
171
|
);
|
|
174
|
-
$headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static(
|
|
172
|
+
$headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static($accept_language));
|
|
175
173
|
#[cfg(feature = "emulation-compression")]
|
|
176
|
-
$headers.insert(
|
|
177
|
-
ACCEPT_ENCODING,
|
|
178
|
-
HeaderValue::from_static("gzip, deflate, br, zstd"),
|
|
179
|
-
);
|
|
174
|
+
$headers.insert(ACCEPT_ENCODING, HeaderValue::from_static($accept_encoding));
|
|
180
175
|
};
|
|
181
176
|
}
|
|
182
177
|
|
|
@@ -50,13 +50,16 @@ macro_rules! tls_options {
|
|
|
50
50
|
.alps_use_new_codepoint(true))
|
|
51
51
|
};
|
|
52
52
|
(9, $curves:expr) => {
|
|
53
|
+
tls_options!(9, $curves, shuffled_trust_anchors(crate::rand::fast_random))
|
|
54
|
+
};
|
|
55
|
+
(9, $curves:expr, $trust_anchors:expr) => {
|
|
53
56
|
tls_options!(@build ChromeTlsConfig::builder()
|
|
54
57
|
.permute_extensions(true)
|
|
55
58
|
.enable_ech_grease(true)
|
|
56
59
|
.pre_shared_key(true)
|
|
57
60
|
.curves($curves)
|
|
58
61
|
.sigalgs_list(NEW_SIGALGS_LIST)
|
|
59
|
-
.trust_anchors(
|
|
62
|
+
.trust_anchors($trust_anchors)
|
|
60
63
|
.grease_sigalgs_enabled(true)
|
|
61
64
|
.alps_use_new_codepoint(true))
|
|
62
65
|
};
|
|
@@ -115,6 +118,44 @@ pub const NEW_SIGALGS_LIST: &str = join!(
|
|
|
115
118
|
// Encoded IDs and wreq's Chromium root store come from the same root set.
|
|
116
119
|
pub(super) const CHROME_TRUST_ANCHORS: &[u8] = &chromium_roots::encoded_trust_anchor_ids();
|
|
117
120
|
|
|
121
|
+
// Chrome 152/153 iterate a salted flat_hash_set held by SSLClientContext, rather than
|
|
122
|
+
// the root store's source order. Generate an order once per native configuration;
|
|
123
|
+
// connections using that client configuration retain the order.
|
|
124
|
+
// https://chromium.googlesource.com/chromium/src/+/f7b831a4e249fbd98eae4b391c69f70300a849d1/net/ssl/ssl_config_service.cc
|
|
125
|
+
pub(super) fn shuffled_trust_anchors(mut random: impl FnMut() -> u64) -> Vec<u8> {
|
|
126
|
+
let mut ids = chromium_roots::trust_anchor_ids();
|
|
127
|
+
for index in (1..ids.len()).rev() {
|
|
128
|
+
let bound = (index + 1) as u64;
|
|
129
|
+
let limit = u64::MAX - u64::MAX % bound;
|
|
130
|
+
let mut value = random();
|
|
131
|
+
while value >= limit {
|
|
132
|
+
value = random();
|
|
133
|
+
}
|
|
134
|
+
ids.swap(index, (value % bound) as usize);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
encode_trust_anchors(&ids)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Chrome 154 sorts the raw IDs before adding their length prefixes.
|
|
141
|
+
// https://chromium.googlesource.com/chromium/src/+/731082f0a26ce4b3976c3d82943092f5d13daf13/net/cert/x509_util.cc
|
|
142
|
+
pub(super) fn sorted_trust_anchors() -> Vec<u8> {
|
|
143
|
+
let mut ids = chromium_roots::trust_anchor_ids();
|
|
144
|
+
ids.sort_unstable();
|
|
145
|
+
encode_trust_anchors(&ids)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
fn encode_trust_anchors(ids: &[&[u8]]) -> Vec<u8> {
|
|
149
|
+
let mut encoded = Vec::with_capacity(CHROME_TRUST_ANCHORS.len());
|
|
150
|
+
for id in ids {
|
|
151
|
+
// encoded_trust_anchor_ids() validates these same IDs at compile time:
|
|
152
|
+
// each length is nonzero and fits in one byte.
|
|
153
|
+
encoded.push(id.len() as u8);
|
|
154
|
+
encoded.extend_from_slice(id);
|
|
155
|
+
}
|
|
156
|
+
encoded
|
|
157
|
+
}
|
|
158
|
+
|
|
118
159
|
pub const CERTIFICATE_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&BrotliCompressor];
|
|
119
160
|
|
|
120
161
|
#[derive(TypedBuilder)]
|
|
@@ -144,7 +185,7 @@ pub struct ChromeTlsConfig {
|
|
|
144
185
|
pre_shared_key: bool,
|
|
145
186
|
|
|
146
187
|
#[builder(default, setter(strip_option))]
|
|
147
|
-
trust_anchors: Option
|
|
188
|
+
trust_anchors: Option<Vec<u8>>,
|
|
148
189
|
|
|
149
190
|
#[builder(default, setter(strip_option))]
|
|
150
191
|
grease_sigalgs_enabled: Option<bool>,
|
|
@@ -181,3 +222,65 @@ impl From<ChromeTlsConfig> for TlsOptions {
|
|
|
181
222
|
builder.build()
|
|
182
223
|
}
|
|
183
224
|
}
|
|
225
|
+
|
|
226
|
+
#[cfg(test)]
|
|
227
|
+
mod tests {
|
|
228
|
+
use super::*;
|
|
229
|
+
use wreq::IntoEmulation;
|
|
230
|
+
|
|
231
|
+
#[test]
|
|
232
|
+
fn chrome_profiles_retain_trust_anchor_order_when_cloned() {
|
|
233
|
+
for profile in [
|
|
234
|
+
crate::Profile::Chrome152,
|
|
235
|
+
crate::Profile::Chrome153,
|
|
236
|
+
crate::Profile::Chrome154,
|
|
237
|
+
] {
|
|
238
|
+
let native = profile.into_emulation();
|
|
239
|
+
let cloned = native.clone();
|
|
240
|
+
let ids = native.tls_options.unwrap().trust_anchors.unwrap();
|
|
241
|
+
assert_eq!(ids.len(), CHROME_TRUST_ANCHORS.len());
|
|
242
|
+
assert_eq!(ids, cloned.tls_options.unwrap().trust_anchors.unwrap());
|
|
243
|
+
let decoded = decode_ids(&ids);
|
|
244
|
+
let mut expected = chromium_roots::trust_anchor_ids();
|
|
245
|
+
expected.sort_unstable();
|
|
246
|
+
if profile == crate::Profile::Chrome154 {
|
|
247
|
+
assert_eq!(decoded, expected);
|
|
248
|
+
}
|
|
249
|
+
let mut actual = decoded;
|
|
250
|
+
actual.sort_unstable();
|
|
251
|
+
assert_eq!(actual, expected);
|
|
252
|
+
}
|
|
253
|
+
assert!(
|
|
254
|
+
crate::Profile::Chrome151
|
|
255
|
+
.into_emulation()
|
|
256
|
+
.tls_options
|
|
257
|
+
.unwrap()
|
|
258
|
+
.trust_anchors
|
|
259
|
+
.is_none()
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
#[test]
|
|
264
|
+
fn trust_anchor_shuffle_preserves_ids_without_pinning_the_first() {
|
|
265
|
+
// Choosing index zero at every step rotates the entire list left.
|
|
266
|
+
// Rejecting u64::MAX also covers the unbiased sampling boundary.
|
|
267
|
+
let mut samples = std::iter::once(u64::MAX).chain(std::iter::repeat(0));
|
|
268
|
+
let encoded = shuffled_trust_anchors(|| samples.next().unwrap());
|
|
269
|
+
let decoded = decode_ids(&encoded);
|
|
270
|
+
let mut expected = chromium_roots::trust_anchor_ids();
|
|
271
|
+
expected.rotate_left(1);
|
|
272
|
+
assert_eq!(decoded, expected);
|
|
273
|
+
assert_eq!(encoded.len(), CHROME_TRUST_ANCHORS.len());
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
fn decode_ids(mut remaining: &[u8]) -> Vec<&[u8]> {
|
|
277
|
+
let mut decoded = Vec::new();
|
|
278
|
+
while let Some((&length, rest)) = remaining.split_first() {
|
|
279
|
+
assert_ne!(length, 0);
|
|
280
|
+
let (id, rest) = rest.split_at(usize::from(length));
|
|
281
|
+
decoded.push(id);
|
|
282
|
+
remaining = rest;
|
|
283
|
+
}
|
|
284
|
+
decoded
|
|
285
|
+
}
|
|
286
|
+
}
|
|
@@ -1929,6 +1929,40 @@ mod_generator!(
|
|
|
1929
1929
|
]
|
|
1930
1930
|
);
|
|
1931
1931
|
|
|
1932
|
+
mod_generator!(
|
|
1933
|
+
v154,
|
|
1934
|
+
tls_options!(9, CURVES_3, sorted_trust_anchors()),
|
|
1935
|
+
http2_options!(3),
|
|
1936
|
+
header_initializer_with_zstd_priority,
|
|
1937
|
+
[
|
|
1938
|
+
(
|
|
1939
|
+
MacOS,
|
|
1940
|
+
r#""Chromium";v="154", "Google Chrome";v="154", "Not A(Brand";v="99""#,
|
|
1941
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36"
|
|
1942
|
+
),
|
|
1943
|
+
(
|
|
1944
|
+
Linux,
|
|
1945
|
+
r#""Chromium";v="154", "Google Chrome";v="154", "Not A(Brand";v="99""#,
|
|
1946
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36"
|
|
1947
|
+
),
|
|
1948
|
+
(
|
|
1949
|
+
Android,
|
|
1950
|
+
r#""Chromium";v="154", "Google Chrome";v="154", "Not A(Brand";v="99""#,
|
|
1951
|
+
"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Mobile Safari/537.36"
|
|
1952
|
+
),
|
|
1953
|
+
(
|
|
1954
|
+
Windows,
|
|
1955
|
+
r#""Chromium";v="154", "Google Chrome";v="154", "Not A(Brand";v="99""#,
|
|
1956
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36"
|
|
1957
|
+
),
|
|
1958
|
+
(
|
|
1959
|
+
IOS,
|
|
1960
|
+
r#""Chromium";v="154", "Google Chrome";v="154", "Not A(Brand";v="99""#,
|
|
1961
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/154.0.8037.41 Mobile/15E148 Safari/604.1"
|
|
1962
|
+
)
|
|
1963
|
+
]
|
|
1964
|
+
);
|
|
1965
|
+
|
|
1932
1966
|
mod_generator!(
|
|
1933
1967
|
edge143,
|
|
1934
1968
|
v132::build_emulation,
|
|
@@ -35,3 +35,25 @@ pub fn header_initializer_with_zstd(ua: &'static str) -> HeaderMap {
|
|
|
35
35
|
);
|
|
36
36
|
headers
|
|
37
37
|
}
|
|
38
|
+
|
|
39
|
+
/// Firefox 147 and later. Identical to [`header_initializer_with_zstd`] apart from the
|
|
40
|
+
/// `Accept-Language` q-value, which Firefox changed in 147 (Bugzilla 2000765).
|
|
41
|
+
pub fn header_initializer_with_zstd_q09(ua: &'static str) -> HeaderMap {
|
|
42
|
+
let mut headers = HeaderMap::new();
|
|
43
|
+
header_firefox_ua!(headers, ua);
|
|
44
|
+
header_firefox_accept!(zstd_q09, headers);
|
|
45
|
+
headers.insert(
|
|
46
|
+
HeaderName::from_static("upgrade-insecure-requests"),
|
|
47
|
+
HeaderValue::from_static("1"),
|
|
48
|
+
);
|
|
49
|
+
header_firefox_sec_fetch!(headers);
|
|
50
|
+
headers.insert(
|
|
51
|
+
HeaderName::from_static("priority"),
|
|
52
|
+
HeaderValue::from_static("u=0, i"),
|
|
53
|
+
);
|
|
54
|
+
headers.insert(
|
|
55
|
+
HeaderName::from_static("te"),
|
|
56
|
+
HeaderValue::from_static("trailers"),
|
|
57
|
+
);
|
|
58
|
+
headers
|
|
59
|
+
}
|
|
@@ -380,7 +380,7 @@ mod_generator!(
|
|
|
380
380
|
mod_generator!(
|
|
381
381
|
ff147,
|
|
382
382
|
ff135::build_emulation,
|
|
383
|
-
|
|
383
|
+
header_initializer_with_zstd_q09,
|
|
384
384
|
[
|
|
385
385
|
(
|
|
386
386
|
Windows,
|
|
@@ -408,7 +408,7 @@ mod_generator!(
|
|
|
408
408
|
mod_generator!(
|
|
409
409
|
ff148,
|
|
410
410
|
ff135::build_emulation,
|
|
411
|
-
|
|
411
|
+
header_initializer_with_zstd_q09,
|
|
412
412
|
[
|
|
413
413
|
(
|
|
414
414
|
Windows,
|
|
@@ -436,7 +436,7 @@ mod_generator!(
|
|
|
436
436
|
mod_generator!(
|
|
437
437
|
ff149,
|
|
438
438
|
ff135::build_emulation,
|
|
439
|
-
|
|
439
|
+
header_initializer_with_zstd_q09,
|
|
440
440
|
[
|
|
441
441
|
(
|
|
442
442
|
Windows,
|
|
@@ -465,7 +465,7 @@ mod_generator!(
|
|
|
465
465
|
ff150,
|
|
466
466
|
tls_options!(4, CIPHER_LIST_3, CURVES_2, KEY_SHARES_2),
|
|
467
467
|
http2_options!(1),
|
|
468
|
-
|
|
468
|
+
header_initializer_with_zstd_q09,
|
|
469
469
|
[
|
|
470
470
|
(
|
|
471
471
|
Windows,
|
|
@@ -493,7 +493,7 @@ mod_generator!(
|
|
|
493
493
|
mod_generator!(
|
|
494
494
|
ff151,
|
|
495
495
|
ff150::build_emulation,
|
|
496
|
-
|
|
496
|
+
header_initializer_with_zstd_q09,
|
|
497
497
|
[
|
|
498
498
|
(
|
|
499
499
|
Windows,
|
|
@@ -521,7 +521,7 @@ mod_generator!(
|
|
|
521
521
|
mod_generator!(
|
|
522
522
|
ff152,
|
|
523
523
|
ff150::build_emulation,
|
|
524
|
-
|
|
524
|
+
header_initializer_with_zstd_q09,
|
|
525
525
|
[
|
|
526
526
|
(
|
|
527
527
|
Windows,
|
|
@@ -64,6 +64,7 @@ define_enum!(
|
|
|
64
64
|
Chrome151 => ("chrome_151", v151::emulation),
|
|
65
65
|
Chrome152 => ("chrome_152", v152::emulation),
|
|
66
66
|
Chrome153 => ("chrome_153", v153::emulation),
|
|
67
|
+
Chrome154 => ("chrome_154", v154::emulation),
|
|
67
68
|
|
|
68
69
|
// Edge versions
|
|
69
70
|
Edge101 => ("edge_101", edge101::emulation),
|
|
@@ -215,6 +216,11 @@ impl Platform {
|
|
|
215
216
|
/// The `Emulation` struct allows you to configure various aspects of profile and platform
|
|
216
217
|
/// emulation, including the profile, platform, and whether to enable certain features
|
|
217
218
|
/// like HTTP/2 or headers.
|
|
219
|
+
///
|
|
220
|
+
/// Conversion through [`wreq::IntoEmulation`] creates a native configuration,
|
|
221
|
+
/// including a randomized Trust Anchor ID order for Chrome 152/153 profiles.
|
|
222
|
+
/// Client-level emulation retains that order for subsequent connections.
|
|
223
|
+
/// Chrome 154 sorts the IDs by their raw bytes instead.
|
|
218
224
|
#[derive(Default, Clone, TypedBuilder)]
|
|
219
225
|
pub struct Emulation {
|
|
220
226
|
/// Whether to change the profile (browser/okhttp) information.
|
|
@@ -295,3 +295,13 @@ test_emulation!(
|
|
|
295
295
|
],
|
|
296
296
|
"52d84b11737d980aef856699f885ca86"
|
|
297
297
|
);
|
|
298
|
+
|
|
299
|
+
test_emulation!(
|
|
300
|
+
test_chrome154,
|
|
301
|
+
Emulation::Chrome154,
|
|
302
|
+
[
|
|
303
|
+
"t13d1517h2_8daaf6152771_cb7bf5808d99",
|
|
304
|
+
"t13d1518h2_8daaf6152771_e2d80978ab2e"
|
|
305
|
+
],
|
|
306
|
+
"52d84b11737d980aef856699f885ca86"
|
|
307
|
+
);
|
|
@@ -111,3 +111,61 @@ test_emulation!(
|
|
|
111
111
|
["t13d1617h2_86a278354501_3cbfd9057e0d"],
|
|
112
112
|
"6ea73faa8fc5aac76bded7bd238f6433"
|
|
113
113
|
);
|
|
114
|
+
|
|
115
|
+
/// Firefox 147 changed the generated `Accept-Language` q-value: the list used to divide 1.0
|
|
116
|
+
/// across its entries, giving `q=0.5` for two, and now decrements by 0.1 per entry, giving
|
|
117
|
+
/// `q=0.9`. See Bugzilla 2000765, landed in mozilla-firefox 0e050ae5116d.
|
|
118
|
+
///
|
|
119
|
+
/// Every profile is listed rather than a range, so adding one is a deliberate choice about
|
|
120
|
+
/// which side of 147 it falls on.
|
|
121
|
+
#[test]
|
|
122
|
+
fn firefox_accept_language_matches_the_profile_version() {
|
|
123
|
+
use wreq::IntoEmulation;
|
|
124
|
+
use wreq_util::{Platform, Profile};
|
|
125
|
+
|
|
126
|
+
const PRE_147: &[(&str, Profile)] = &[
|
|
127
|
+
("Firefox109", Emulation::Firefox109),
|
|
128
|
+
("Firefox117", Emulation::Firefox117),
|
|
129
|
+
("Firefox128", Emulation::Firefox128),
|
|
130
|
+
("Firefox133", Emulation::Firefox133),
|
|
131
|
+
("Firefox135", Emulation::Firefox135),
|
|
132
|
+
("FirefoxPrivate135", Emulation::FirefoxPrivate135),
|
|
133
|
+
("FirefoxAndroid135", Emulation::FirefoxAndroid135),
|
|
134
|
+
("Firefox136", Emulation::Firefox136),
|
|
135
|
+
("FirefoxPrivate136", Emulation::FirefoxPrivate136),
|
|
136
|
+
("Firefox139", Emulation::Firefox139),
|
|
137
|
+
("Firefox142", Emulation::Firefox142),
|
|
138
|
+
("Firefox143", Emulation::Firefox143),
|
|
139
|
+
("Firefox144", Emulation::Firefox144),
|
|
140
|
+
("Firefox145", Emulation::Firefox145),
|
|
141
|
+
("Firefox146", Emulation::Firefox146),
|
|
142
|
+
];
|
|
143
|
+
const POST_147: &[(&str, Profile)] = &[
|
|
144
|
+
("Firefox147", Emulation::Firefox147),
|
|
145
|
+
("Firefox148", Emulation::Firefox148),
|
|
146
|
+
("Firefox149", Emulation::Firefox149),
|
|
147
|
+
("Firefox150", Emulation::Firefox150),
|
|
148
|
+
("Firefox151", Emulation::Firefox151),
|
|
149
|
+
("Firefox152", Emulation::Firefox152),
|
|
150
|
+
];
|
|
151
|
+
|
|
152
|
+
fn accept_language(profile: Profile) -> String {
|
|
153
|
+
Emulation::builder()
|
|
154
|
+
.profile(profile)
|
|
155
|
+
.platform(Platform::Linux)
|
|
156
|
+
.build()
|
|
157
|
+
.into_emulation()
|
|
158
|
+
.headers
|
|
159
|
+
.get("accept-language")
|
|
160
|
+
.and_then(|value| value.to_str().ok())
|
|
161
|
+
.unwrap_or_default()
|
|
162
|
+
.to_owned()
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
for (name, profile) in PRE_147 {
|
|
166
|
+
assert_eq!("en-US,en;q=0.5", accept_language(*profile), "{name}");
|
|
167
|
+
}
|
|
168
|
+
for (name, profile) in POST_147 {
|
|
169
|
+
assert_eq!("en-US,en;q=0.9", accept_language(*profile), "{name}");
|
|
170
|
+
}
|
|
171
|
+
}
|
data/lib/wreq_ruby/emulate.rb
CHANGED
|
@@ -65,6 +65,7 @@ module Wreq
|
|
|
65
65
|
Chrome151 = nil
|
|
66
66
|
Chrome152 = nil
|
|
67
67
|
Chrome153 = nil
|
|
68
|
+
Chrome154 = nil
|
|
68
69
|
|
|
69
70
|
Edge101 = nil
|
|
70
71
|
Edge122 = nil
|
|
@@ -263,6 +264,14 @@ module Wreq
|
|
|
263
264
|
# for HTTP/2 and automatic default headers. The actual implementation is
|
|
264
265
|
# provided by Rust.
|
|
265
266
|
#
|
|
267
|
+
# Construction generates the native configuration once. For Chrome 152/153,
|
|
268
|
+
# the TLS trust_anchors extension's ID order is randomized at construction;
|
|
269
|
+
# Chrome 154 sorts those IDs by their raw bytes instead. Reusing this object
|
|
270
|
+
# across clients or requests retains that order. Creating a new object generates
|
|
271
|
+
# a new configuration, including a fresh shuffle for Chrome 152/153.
|
|
272
|
+
# This does not freeze other per-handshake randomness or force a new handshake
|
|
273
|
+
# when a request reuses an existing connection.
|
|
274
|
+
#
|
|
266
275
|
# `profile:` defaults to the library's default profile when omitted.
|
|
267
276
|
# `platform:` defaults to the library's default platform when omitted.
|
|
268
277
|
#
|
|
@@ -288,6 +297,10 @@ module Wreq
|
|
|
288
297
|
# Native fields and methods are set by the extension.
|
|
289
298
|
# This stub is for documentation only.
|
|
290
299
|
unless singleton_methods(false).include?(:new)
|
|
300
|
+
# Generates a reusable native configuration. Chrome 152/153 Trust Anchor ID
|
|
301
|
+
# order is chosen here and retained when this object is reused; Chrome 154
|
|
302
|
+
# uses raw-byte sorted order. See {Emulation} for connection semantics.
|
|
303
|
+
#
|
|
291
304
|
# @param profile [Wreq::Profile, nil] Fingerprint profile to emulate
|
|
292
305
|
# @param platform [Wreq::Platform, nil] Operating system platform to emulate
|
|
293
306
|
# @param http2 [Boolean, nil] Whether HTTP/2 emulation is enabled; defaults
|
data/src/emulate.rs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
use ::serde::Deserialize;
|
|
2
2
|
use magnus::{Error, Module, Object, RModule, Ruby, Value, function, method, typed_data::Obj};
|
|
3
|
+
use wreq::IntoEmulation;
|
|
3
4
|
|
|
4
5
|
use crate::options::{NativeOption, Options};
|
|
5
6
|
|
|
@@ -90,6 +91,7 @@ define_ruby_enum!(
|
|
|
90
91
|
Chrome151,
|
|
91
92
|
Chrome152,
|
|
92
93
|
Chrome153,
|
|
94
|
+
Chrome154,
|
|
93
95
|
|
|
94
96
|
Edge101,
|
|
95
97
|
Edge122,
|
|
@@ -205,10 +207,13 @@ define_ruby_enum!(
|
|
|
205
207
|
IOS => "ios",
|
|
206
208
|
);
|
|
207
209
|
|
|
208
|
-
///
|
|
210
|
+
/// Materialized emulation configuration owned by the Ruby object.
|
|
211
|
+
///
|
|
212
|
+
/// Clients and requests clone this value rather than regenerating profile settings,
|
|
213
|
+
/// preserving configuration-level randomness for the lifetime of the object.
|
|
209
214
|
#[derive(Clone)]
|
|
210
215
|
#[magnus::wrap(class = "Wreq::Emulation", free_immediately, size)]
|
|
211
|
-
pub struct Emulation(pub
|
|
216
|
+
pub struct Emulation(pub wreq::Emulation);
|
|
212
217
|
|
|
213
218
|
// ===== impl Emulation =====
|
|
214
219
|
|
|
@@ -245,7 +250,8 @@ impl Emulation {
|
|
|
245
250
|
)
|
|
246
251
|
.http2(params.http2.unwrap_or(true))
|
|
247
252
|
.headers(params.headers.unwrap_or(true))
|
|
248
|
-
.build()
|
|
253
|
+
.build()
|
|
254
|
+
.into_emulation();
|
|
249
255
|
|
|
250
256
|
Ok(Self(emulation))
|
|
251
257
|
}
|
data/test/emulation_test.rb
CHANGED
|
@@ -1,8 +1,28 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "test_helper"
|
|
4
|
+
require "socket"
|
|
5
|
+
require "stringio"
|
|
6
|
+
require "timeout"
|
|
4
7
|
|
|
5
8
|
class EmulationTest < Minitest::Test
|
|
9
|
+
def test_reused_emulation_retains_trust_anchors_across_clients_and_requests
|
|
10
|
+
[Wreq::Profile::Chrome152, Wreq::Profile::Chrome153, Wreq::Profile::Chrome154].each do |profile|
|
|
11
|
+
emulation = Wreq::Emulation.new(profile: profile)
|
|
12
|
+
client = Wreq::Client.new(emulation: emulation, no_proxy: true)
|
|
13
|
+
expected = capture_trust_anchors { |url| client.get(url, timeout: 5) }
|
|
14
|
+
refute_empty expected, profile.to_s
|
|
15
|
+
|
|
16
|
+
request_client = Wreq::Client.new(no_proxy: true)
|
|
17
|
+
2.times do
|
|
18
|
+
actual = capture_trust_anchors do |url|
|
|
19
|
+
request_client.get(url, emulation: emulation, timeout: 5)
|
|
20
|
+
end
|
|
21
|
+
assert_equal expected, actual, "#{profile}: reusing an object must retain its configuration"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
6
26
|
def test_all_emulation_device_constants_are_non_nil
|
|
7
27
|
Wreq::Profile.constants.each do |name|
|
|
8
28
|
const = Wreq::Profile.const_get(name)
|
|
@@ -22,6 +42,7 @@ class EmulationTest < Minitest::Test
|
|
|
22
42
|
{
|
|
23
43
|
Wreq::Profile::Chrome152 => "Chrome152",
|
|
24
44
|
Wreq::Profile::Chrome153 => "Chrome153",
|
|
45
|
+
Wreq::Profile::Chrome154 => "Chrome154",
|
|
25
46
|
Wreq::Profile::Firefox152 => "Firefox152"
|
|
26
47
|
}.each do |profile, name|
|
|
27
48
|
assert_equal name, profile.to_s
|
|
@@ -36,4 +57,60 @@ class EmulationTest < Minitest::Test
|
|
|
36
57
|
"#{name} should be Platform, got #{const.inspect}"
|
|
37
58
|
end
|
|
38
59
|
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
# Each listener forces a new connection. Stop after ClientHello so this test
|
|
64
|
+
# inspects the extension bytes without depending on certificates or the network.
|
|
65
|
+
def capture_trust_anchors
|
|
66
|
+
server = TCPServer.new("127.0.0.1", 0)
|
|
67
|
+
thread = Thread.new do
|
|
68
|
+
socket = nil
|
|
69
|
+
Timeout.timeout(5) do
|
|
70
|
+
socket = server.accept
|
|
71
|
+
handshake = +"".b
|
|
72
|
+
loop do
|
|
73
|
+
header = read_tls_bytes(socket, 5)
|
|
74
|
+
raise "expected a TLS handshake record" unless header.getbyte(0) == 22
|
|
75
|
+
handshake << read_tls_bytes(socket, header.byteslice(3, 2).unpack1("n"))
|
|
76
|
+
next if handshake.bytesize < 4
|
|
77
|
+
length = handshake.byteslice(1, 3).bytes.reduce(0) { |n, byte| (n << 8) | byte }
|
|
78
|
+
break if handshake.bytesize >= length + 4
|
|
79
|
+
end
|
|
80
|
+
trust_anchors_from_client_hello(handshake)
|
|
81
|
+
end
|
|
82
|
+
ensure
|
|
83
|
+
socket&.close
|
|
84
|
+
end
|
|
85
|
+
thread.report_on_exception = false
|
|
86
|
+
|
|
87
|
+
assert_raises(Wreq::ConnectionError) { yield "https://127.0.0.1:#{server.addr[1]}/" }
|
|
88
|
+
thread.value
|
|
89
|
+
ensure
|
|
90
|
+
server&.close
|
|
91
|
+
thread&.kill if thread&.alive?
|
|
92
|
+
thread&.join
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def read_tls_bytes(io, length)
|
|
96
|
+
bytes = io.read(length)
|
|
97
|
+
raise EOFError, "truncated ClientHello" unless bytes&.bytesize == length
|
|
98
|
+
bytes
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def trust_anchors_from_client_hello(handshake)
|
|
102
|
+
io = StringIO.new(handshake)
|
|
103
|
+
raise "expected ClientHello" unless read_tls_bytes(io, 1).unpack1("C") == 1
|
|
104
|
+
read_tls_bytes(io, 3 + 2 + 32) # Handshake length, legacy version and random.
|
|
105
|
+
read_tls_bytes(io, read_tls_bytes(io, 1).unpack1("C")) # Session ID.
|
|
106
|
+
read_tls_bytes(io, read_tls_bytes(io, 2).unpack1("n")) # Cipher suites.
|
|
107
|
+
read_tls_bytes(io, read_tls_bytes(io, 1).unpack1("C")) # Compression methods.
|
|
108
|
+
extensions = StringIO.new(read_tls_bytes(io, read_tls_bytes(io, 2).unpack1("n")))
|
|
109
|
+
until extensions.eof?
|
|
110
|
+
type, length = read_tls_bytes(extensions, 4).unpack("nn")
|
|
111
|
+
data = read_tls_bytes(extensions, length)
|
|
112
|
+
return data if type == 51764
|
|
113
|
+
end
|
|
114
|
+
raise "missing trust_anchors extension"
|
|
115
|
+
end
|
|
39
116
|
end
|