http_mimic 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +26 -15
- data/lib/http_mimic/command_builder.rb +74 -35
- data/lib/http_mimic/configuration.rb +17 -14
- data/lib/http_mimic/request.rb +16 -4
- data/lib/http_mimic/response.rb +84 -0
- data/lib/http_mimic/response_parser.rb +45 -21
- data/lib/http_mimic/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 11e55edf4c06160f27621a651e4c16eae929ec08767b5b6eff6b007d8a3a1d4b
|
|
4
|
+
data.tar.gz: e90137a02ee867cd744641c725cd3daef2994b5f4a7216304f7ca189e6d92303
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 77dfc30ee87e63105f5cbf95e818cc4b01841f9862566fdceedad277cb22366c05c37ab14b3ac699ddb30d68a9ead35acc43114a1f882c034360d06807450677
|
|
7
|
+
data.tar.gz: 4ab621140feebc4c3f9e8897d33edf510785ea6c5cbad5cf08b672ecfde52f6146b6f1a3d695f5129acce0d3c7b64cfdae275803480178500e9c683fb7c053a8
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.3.2] - 2026-08-29
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **Mobile Impersonation & Multi-Stage Auto-Fallback**:
|
|
12
|
+
- Added support for mobile browser targets: Android Chrome (`chrome131_android`, `chrome99_android`, alias `android`) and iOS Safari (`safari260_ios`, `safari180_ios`, `safari184_ios`, `safari172_ios`, alias `ios`, `mobile`).
|
|
13
|
+
- Enhanced `:auto` adaptive mode with automatic fallback cascade `[:impersonate, :android, :ios, :curl]` to bypass modern anti-bot systems (e.g. Akamai Bot Manager on Adidas HK) without requiring JavaScript telemetry solving.
|
|
14
|
+
- Added execution modes: `:mobile_first`, `:android_first`, `:ios_first`, `:mobile_only`, `:android_only`, `:ios_only`.
|
|
15
|
+
- **Binary-Safe Response Parsing & File Saving**:
|
|
16
|
+
- Fixed binary handling in `ResponseParser#split_headers_and_body` to prevent `ArgumentError: invalid byte sequence in UTF-8` on images, PDFs, compressed files, and binary streams.
|
|
17
|
+
- Added `response.binary?` and `response.save_to_file(path)` / `response.save(path)` helpers.
|
|
18
|
+
- **Built-in HTML / Scraping Helpers**:
|
|
19
|
+
- Added `response.title`, `response.og_image`, `response.meta_description`, and `response.extract_images(base_url: nil)`.
|
|
20
|
+
|
|
8
21
|
## [0.3.1] - 2026-08-25
|
|
9
22
|
|
|
10
23
|
### Added
|
data/README.md
CHANGED
|
@@ -161,24 +161,27 @@ response = client.post('/v1/users', json: { username: 'bob' })
|
|
|
161
161
|
|
|
162
162
|
## 🧠 Smart Adaptive Modes (Zero-Configuration Scraping)
|
|
163
163
|
|
|
164
|
-
`HttpMimic` provides built-in multi-strategy orchestration so you can fetch protected websites without worrying about which specific WAF (Cloudflare,
|
|
164
|
+
`HttpMimic` provides built-in multi-strategy orchestration so you can fetch protected websites without worrying about which specific WAF (Akamai Botman, Cloudflare Turnstile, DataDome, Kasada) protects them:
|
|
165
165
|
|
|
166
|
-
- **`:auto` (Default & Recommended)**: Tries `curl-impersonate` (
|
|
167
|
-
- **`:
|
|
168
|
-
- **`:
|
|
169
|
-
- **`:
|
|
170
|
-
- **`:curl_only`**: Strictly uses
|
|
166
|
+
- **`:auto` (Default & Recommended)**: Tries desktop `curl-impersonate` (`chrome131`) first. If blocked by WAFs like Akamai with `403`/`429`/`503` (which require JS telemetry for desktop browsers), it **automatically cascades through mobile profiles (`android`, `ios`) and server `curl` headers**, reliably retrieving `200 OK`.
|
|
167
|
+
- **`:mobile_first` / `:android_first` / `:ios_first`**: Prefers mobile TLS/HTTP2 fingerprints and falls back to desktop or curl if blocked.
|
|
168
|
+
- **`:impersonate_first`**: Prefers desktop `curl-impersonate` and automatically falls back to mobile and curl if blocked.
|
|
169
|
+
- **`:curl_first`**: Prefers standard server `curl` and automatically upgrades to impersonate profiles if blocked.
|
|
170
|
+
- **`:impersonate_only` / `:mobile_only` / `:curl_only`**: Strictly uses the chosen profile (no fallback).
|
|
171
171
|
|
|
172
172
|
```ruby
|
|
173
|
-
# 1. No-Brain Auto Mode (Works automatically for
|
|
174
|
-
response = HttpMimic.get('https://www.
|
|
173
|
+
# 1. No-Brain Auto Mode (Works automatically for Cloudflare, Akamai, etc.):
|
|
174
|
+
response = HttpMimic.get('https://www.adidas.com.hk/en/KI8139.html')
|
|
175
175
|
puts response.code # => 200
|
|
176
|
-
puts response.mode_used # => :
|
|
176
|
+
puts response.mode_used # => :android
|
|
177
177
|
puts response.fallback_triggered? # => true
|
|
178
178
|
|
|
179
|
-
# 2.
|
|
180
|
-
response
|
|
181
|
-
|
|
179
|
+
# 2. Extract images and metadata with built-in helpers:
|
|
180
|
+
puts response.title # => "SAMBA OG SHOES - Brown | adidas Hong Kong"
|
|
181
|
+
images = response.extract_images # => ["https://assets.adidas.com/images/.../KI8139_01_00_standard.jpg", ...]
|
|
182
|
+
|
|
183
|
+
# 3. Fetch and save binary images directly:
|
|
184
|
+
HttpMimic.get(images.first).save('samba_og.jpg')
|
|
182
185
|
```
|
|
183
186
|
|
|
184
187
|
---
|
|
@@ -257,18 +260,26 @@ response.redirect? # => false (3xx)
|
|
|
257
260
|
response.client_error? # => false (4xx)
|
|
258
261
|
response.server_error? # => false (5xx)
|
|
259
262
|
|
|
260
|
-
# Response body
|
|
261
|
-
response.body # => Raw Body (String)
|
|
263
|
+
# Response body & File operations
|
|
264
|
+
response.body # => Raw Body (String / binary bytes)
|
|
265
|
+
response.binary? # => true if content is an image, pdf, or binary stream
|
|
266
|
+
response.save('image.jpg') # => Directly saves response body to file
|
|
262
267
|
response.parsed_response# => Auto-parsed JSON Hash / Array
|
|
263
268
|
response['key'] # => Direct key access to parsed_response
|
|
264
269
|
|
|
270
|
+
# HTML & Scraping Helpers
|
|
271
|
+
response.title # => HTML page title
|
|
272
|
+
response.og_image # => OpenGraph image URL
|
|
273
|
+
response.meta_description # => Meta description
|
|
274
|
+
response.extract_images # => Array of all resolved image URLs in HTML
|
|
275
|
+
|
|
265
276
|
# Headers & Cookies
|
|
266
277
|
response.headers['content-type'] # => Case-insensitive header access
|
|
267
278
|
response.cookies['session_id'] # => Parsed Set-Cookie store
|
|
268
279
|
response.history # => Array of redirect history metadata
|
|
269
280
|
|
|
270
281
|
# Underlying execution & multi-strategy details
|
|
271
|
-
response.mode_used # => :impersonate or :curl
|
|
282
|
+
response.mode_used # => :impersonate, :android, :ios, or :curl
|
|
272
283
|
response.fallback_triggered? # => true if smart fallback was executed
|
|
273
284
|
response.attempts # => Array of execution metadata for each attempt
|
|
274
285
|
response.exit_code # => Process exit status (0 for success)
|
|
@@ -14,40 +14,70 @@ module HttpMimic
|
|
|
14
14
|
].freeze
|
|
15
15
|
|
|
16
16
|
TARGET_BINARY_MAP = {
|
|
17
|
-
|
|
18
|
-
'
|
|
19
|
-
'
|
|
20
|
-
'
|
|
21
|
-
'
|
|
22
|
-
'
|
|
23
|
-
'
|
|
24
|
-
'
|
|
25
|
-
'
|
|
26
|
-
'
|
|
27
|
-
'
|
|
28
|
-
'
|
|
29
|
-
'
|
|
30
|
-
'
|
|
31
|
-
'
|
|
32
|
-
'
|
|
33
|
-
'
|
|
34
|
-
'
|
|
35
|
-
'
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
'
|
|
39
|
-
'
|
|
40
|
-
'
|
|
41
|
-
'
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
'
|
|
45
|
-
'
|
|
46
|
-
'
|
|
47
|
-
'
|
|
48
|
-
'
|
|
49
|
-
'
|
|
50
|
-
'
|
|
17
|
+
# Chrome Desktop
|
|
18
|
+
'chrome' => %w[curl_chrome131 curl_chrome124 curl_chrome120 curl_chrome116 curl_chrome110 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
19
|
+
'chrome116' => %w[curl_chrome116 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
20
|
+
'chrome120' => %w[curl_chrome120 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
21
|
+
'chrome123' => %w[curl_chrome123 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
22
|
+
'chrome124' => %w[curl_chrome124 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
23
|
+
'chrome131' => %w[curl_chrome131 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
24
|
+
'chrome133a' => %w[curl_chrome133a curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
25
|
+
'chrome136' => %w[curl_chrome136 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
26
|
+
'chrome142' => %w[curl_chrome142 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
27
|
+
'chrome145' => %w[curl_chrome145 curl_chrome146 curl_chrome150 curl_chrome131 curl-impersonate],
|
|
28
|
+
'chrome146' => %w[curl_chrome146 curl_chrome150 curl_chrome145 curl_chrome131 curl-impersonate],
|
|
29
|
+
'chrome150' => %w[curl_chrome150 curl_chrome146 curl_chrome145 curl_chrome131 curl-impersonate],
|
|
30
|
+
'chrome110' => %w[curl_chrome110 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
31
|
+
'chrome107' => %w[curl_chrome107 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
32
|
+
'chrome104' => %w[curl_chrome104 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
33
|
+
'chrome101' => %w[curl_chrome101 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
34
|
+
'chrome100' => %w[curl_chrome100 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
35
|
+
'chrome99' => %w[curl_chrome99 curl-impersonate-chrome curl_chrome curl-impersonate],
|
|
36
|
+
|
|
37
|
+
# Mobile / Android
|
|
38
|
+
'chrome131_android' => %w[curl_chrome131_android curl_chrome131 curl-impersonate-chrome curl-impersonate],
|
|
39
|
+
'chrome99_android' => %w[curl_chrome99_android curl_chrome99 curl-impersonate-chrome curl-impersonate],
|
|
40
|
+
'android' => %w[curl_chrome131_android curl_safari180_ios curl_chrome99_android curl_chrome131 curl-impersonate],
|
|
41
|
+
'chrome_android' => %w[curl_chrome131_android curl_chrome99_android curl_chrome131 curl-impersonate],
|
|
42
|
+
|
|
43
|
+
# Firefox
|
|
44
|
+
'firefox' => %w[curl_firefox135 curl_firefox133 curl_ff117 curl_firefox117 curl_ff109 curl-impersonate-ff curl_firefox curl-impersonate],
|
|
45
|
+
'firefox147' => %w[curl_firefox147 curl_firefox144 curl_firefox135 curl-impersonate-ff curl-impersonate],
|
|
46
|
+
'firefox144' => %w[curl_firefox144 curl_firefox147 curl_firefox135 curl-impersonate-ff curl-impersonate],
|
|
47
|
+
'firefox135' => %w[curl_firefox135 curl-impersonate-ff curl_firefox curl-impersonate],
|
|
48
|
+
'firefox133' => %w[curl_firefox133 curl-impersonate-ff curl_firefox curl-impersonate],
|
|
49
|
+
'firefox117' => %w[curl_ff117 curl_firefox117 curl-impersonate-ff curl_firefox curl-impersonate],
|
|
50
|
+
'firefox109' => %w[curl_ff109 curl_firefox109 curl-impersonate-ff curl_firefox curl-impersonate],
|
|
51
|
+
'firefox102' => %w[curl_ff102 curl_firefox102 curl-impersonate-ff curl_firefox curl-impersonate],
|
|
52
|
+
'firefox98' => %w[curl_ff98 curl_firefox98 curl-impersonate-ff curl_firefox curl-impersonate],
|
|
53
|
+
|
|
54
|
+
# Safari Desktop
|
|
55
|
+
'safari' => %w[curl_safari180 curl_safari170 curl_safari184 curl_safari260 curl_safari155 curl_safari153 curl-impersonate-safari curl_safari curl-impersonate],
|
|
56
|
+
'safari2601' => %w[curl_safari2601 curl_safari260 curl_safari180 curl-impersonate-safari curl-impersonate],
|
|
57
|
+
'safari260' => %w[curl_safari260 curl_safari2601 curl_safari180 curl-impersonate-safari curl-impersonate],
|
|
58
|
+
'safari184' => %w[curl_safari184 curl_safari180 curl-impersonate-safari curl-impersonate],
|
|
59
|
+
'safari180' => %w[curl_safari180 curl_safari18_0 curl-impersonate-safari curl_safari curl-impersonate],
|
|
60
|
+
'safari170' => %w[curl_safari170 curl_safari17_0 curl-impersonate-safari curl_safari curl-impersonate],
|
|
61
|
+
'safari155' => %w[curl_safari155 curl_safari15_5 curl-impersonate-safari curl_safari curl-impersonate],
|
|
62
|
+
'safari15_5' => %w[curl_safari155 curl_safari15_5 curl-impersonate-safari curl_safari curl-impersonate],
|
|
63
|
+
'safari153' => %w[curl_safari153 curl_safari15_3 curl-impersonate-safari curl_safari curl-impersonate],
|
|
64
|
+
'safari15_3' => %w[curl_safari153 curl_safari15_3 curl-impersonate-safari curl_safari curl-impersonate],
|
|
65
|
+
|
|
66
|
+
# Safari iOS / Mobile
|
|
67
|
+
'safari180_ios' => %w[curl_safari180_ios curl_safari184_ios curl_safari260_ios curl_safari172_ios curl_safari180 curl-impersonate],
|
|
68
|
+
'safari184_ios' => %w[curl_safari184_ios curl_safari180_ios curl_safari260_ios curl_safari172_ios curl_safari180 curl-impersonate],
|
|
69
|
+
'safari260_ios' => %w[curl_safari260_ios curl_safari184_ios curl_safari180_ios curl_safari172_ios curl_safari180 curl-impersonate],
|
|
70
|
+
'safari172_ios' => %w[curl_safari172_ios curl_safari180_ios curl_safari184_ios curl_safari180 curl-impersonate],
|
|
71
|
+
'ios' => %w[curl_safari180_ios curl_safari184_ios curl_safari260_ios curl_safari172_ios curl_safari180 curl-impersonate],
|
|
72
|
+
'safari_ios' => %w[curl_safari180_ios curl_safari184_ios curl_safari260_ios curl_safari172_ios curl_safari180 curl-impersonate],
|
|
73
|
+
'mobile' => %w[curl_safari180_ios curl_chrome131_android curl_safari184_ios curl_safari260_ios curl-impersonate],
|
|
74
|
+
|
|
75
|
+
# Edge & Tor
|
|
76
|
+
'edge' => %w[curl_edge101 curl_edge99 curl-impersonate-edge curl_edge curl-impersonate],
|
|
77
|
+
'edge101' => %w[curl_edge101 curl-impersonate-edge curl_edge curl-impersonate],
|
|
78
|
+
'edge99' => %w[curl_edge99 curl-impersonate-edge curl_edge curl-impersonate],
|
|
79
|
+
'tor' => %w[curl_tor145 curl_tor curl-impersonate],
|
|
80
|
+
'tor145' => %w[curl_tor145 curl_tor curl-impersonate]
|
|
51
81
|
}.freeze
|
|
52
82
|
|
|
53
83
|
attr_reader :method, :url, :options, :config
|
|
@@ -218,7 +248,16 @@ module HttpMimic
|
|
|
218
248
|
raise BinaryNotFoundError, "Specified executable does not exist or is not executable: #{explicit}"
|
|
219
249
|
end
|
|
220
250
|
|
|
221
|
-
target =
|
|
251
|
+
target = case options[:profile]
|
|
252
|
+
when :android
|
|
253
|
+
(options[:android_impersonate] || 'chrome131_android').to_s.downcase
|
|
254
|
+
when :ios
|
|
255
|
+
(options[:ios_impersonate] || 'safari260_ios').to_s.downcase
|
|
256
|
+
when :mobile
|
|
257
|
+
(options[:mobile_impersonate] || 'chrome131_android').to_s.downcase
|
|
258
|
+
else
|
|
259
|
+
(options[:impersonate] || config.default_impersonate).to_s.downcase
|
|
260
|
+
end
|
|
222
261
|
candidate_names = TARGET_BINARY_MAP[target] || ["curl_#{target}", target]
|
|
223
262
|
|
|
224
263
|
# 2. Check local installation directory (~/.http_mimic/bin)
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
module HttpMimic
|
|
4
4
|
class Configuration
|
|
5
|
+
# Browser simulation & request defaults
|
|
5
6
|
attr_accessor :default_impersonate
|
|
7
|
+
attr_accessor :default_mobile_impersonate
|
|
6
8
|
attr_accessor :binary_path
|
|
7
9
|
attr_accessor :fallback_to_curl
|
|
8
10
|
attr_accessor :default_timeout
|
|
@@ -25,21 +27,22 @@ module HttpMimic
|
|
|
25
27
|
attr_accessor :github_repo
|
|
26
28
|
|
|
27
29
|
def initialize
|
|
28
|
-
@default_impersonate
|
|
29
|
-
@
|
|
30
|
-
@
|
|
31
|
-
@
|
|
32
|
-
@
|
|
33
|
-
@
|
|
34
|
-
@
|
|
35
|
-
@
|
|
36
|
-
@
|
|
37
|
-
@
|
|
30
|
+
@default_impersonate = 'chrome131'
|
|
31
|
+
@default_mobile_impersonate = 'safari180_ios'
|
|
32
|
+
@binary_path = nil
|
|
33
|
+
@fallback_to_curl = true
|
|
34
|
+
@default_timeout = 30
|
|
35
|
+
@default_connect_timeout = 10
|
|
36
|
+
@follow_redirects = true
|
|
37
|
+
@max_redirects = 10
|
|
38
|
+
@raise_on_error = false
|
|
39
|
+
@logger = nil
|
|
40
|
+
@debug = false
|
|
38
41
|
|
|
39
|
-
# Multi-strategy defaults: :auto seamlessly falls back between impersonate and curl
|
|
40
|
-
@mode
|
|
41
|
-
@auto_fallback
|
|
42
|
-
@retry_statuses
|
|
42
|
+
# Multi-strategy defaults: :auto seamlessly falls back between impersonate, mobile, and curl
|
|
43
|
+
@mode = :auto
|
|
44
|
+
@auto_fallback = true
|
|
45
|
+
@retry_statuses = [403, 429, 503]
|
|
43
46
|
|
|
44
47
|
# Enable automatic downloading of curl-impersonate driver (lexiforest) by default
|
|
45
48
|
@auto_download = true
|
data/lib/http_mimic/request.rb
CHANGED
|
@@ -83,17 +83,29 @@ module HttpMimic
|
|
|
83
83
|
def determine_profiles(mode, auto_fallback)
|
|
84
84
|
case mode
|
|
85
85
|
when :curl, :curl_first
|
|
86
|
-
auto_fallback ? [:curl, :impersonate] : [:curl]
|
|
86
|
+
auto_fallback ? [:curl, :impersonate, :android, :ios] : [:curl]
|
|
87
87
|
when :curl_only
|
|
88
88
|
[:curl]
|
|
89
89
|
when :impersonate_only
|
|
90
90
|
[:impersonate]
|
|
91
|
+
when :mobile_only
|
|
92
|
+
[:mobile]
|
|
93
|
+
when :android_only
|
|
94
|
+
[:android]
|
|
95
|
+
when :ios_only
|
|
96
|
+
[:ios]
|
|
97
|
+
when :mobile_first
|
|
98
|
+
auto_fallback ? [:android, :ios, :impersonate, :curl] : [:mobile]
|
|
99
|
+
when :android_first
|
|
100
|
+
auto_fallback ? [:android, :ios, :impersonate, :curl] : [:android]
|
|
101
|
+
when :ios_first
|
|
102
|
+
auto_fallback ? [:ios, :android, :impersonate, :curl] : [:ios]
|
|
91
103
|
when :impersonate_first
|
|
92
|
-
auto_fallback ? [:impersonate, :curl] : [:impersonate]
|
|
104
|
+
auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
|
|
93
105
|
when :auto, :smart
|
|
94
|
-
auto_fallback ? [:impersonate, :curl] : [:impersonate]
|
|
106
|
+
auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
|
|
95
107
|
else
|
|
96
|
-
auto_fallback ? [:impersonate, :curl] : [:impersonate]
|
|
108
|
+
auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
|
|
97
109
|
end
|
|
98
110
|
end
|
|
99
111
|
|
data/lib/http_mimic/response.rb
CHANGED
|
@@ -67,6 +67,76 @@ module HttpMimic
|
|
|
67
67
|
client_error? || server_error?
|
|
68
68
|
end
|
|
69
69
|
|
|
70
|
+
def binary?
|
|
71
|
+
content_type = headers['content-type'].to_s.downcase
|
|
72
|
+
ResponseParser::BINARY_MIME_KEYWORDS.any? { |kw| content_type.include?(kw) } ||
|
|
73
|
+
body.to_s.b[0, 1024]&.include?("\x00".b)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def save_to_file(filepath)
|
|
77
|
+
require 'fileutils'
|
|
78
|
+
FileUtils.mkdir_p(File.dirname(filepath))
|
|
79
|
+
File.binwrite(filepath, body)
|
|
80
|
+
filepath
|
|
81
|
+
end
|
|
82
|
+
alias save save_to_file
|
|
83
|
+
|
|
84
|
+
def title
|
|
85
|
+
return nil if binary?
|
|
86
|
+
body[/<title[^>]*>(.*?)<\/title>/im, 1]&.strip
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def og_image
|
|
90
|
+
return nil if binary?
|
|
91
|
+
body[/<meta\s+[^>]*property=["']og:image["'][^>]*content=["']([^"']+)["']/im, 1] ||
|
|
92
|
+
body[/<meta\s+[^>]*content=["']([^"']+)["'][^>]*property=["']og:image["']/im, 1]
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def meta_description
|
|
96
|
+
return nil if binary?
|
|
97
|
+
body[/<meta\s+[^>]*name=["']description["'][^>]*content=["']([^"']+)["']/im, 1] ||
|
|
98
|
+
body[/<meta\s+[^>]*content=["']([^"']+)["'][^>]*name=["']description["']/im, 1]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def extract_images(base_url: nil)
|
|
102
|
+
return [] if binary?
|
|
103
|
+
|
|
104
|
+
base = base_url || request_url || ''
|
|
105
|
+
images = []
|
|
106
|
+
|
|
107
|
+
# 1. Match img tags (src, data-src, data-zoom-image, data-original, srcset)
|
|
108
|
+
body.scan(/<img\s+[^>]*>/i).each do |img_tag|
|
|
109
|
+
%w[src data-src data-zoom-image data-original data-high-res-src data-full-size-image-url].each do |attr|
|
|
110
|
+
if match = img_tag.match(/#{attr}=["']([^"']+)["']/i)
|
|
111
|
+
images << match[1].strip
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
if match = img_tag.match(/srcset=["']([^"']+)["']/i)
|
|
116
|
+
match[1].split(',').each do |item|
|
|
117
|
+
url = item.strip.split(/\s+/).first
|
|
118
|
+
images << url if url && !url.empty?
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# 2. Match og:image meta tag
|
|
124
|
+
if og = og_image
|
|
125
|
+
images << og
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# 3. Match raw image URLs found in document / script payloads
|
|
129
|
+
body.scan(/https?:[^\s"'<>]+\.(?:jpg|jpeg|png|webp|avif|gif)/i).each do |raw_url|
|
|
130
|
+
images << raw_url
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Clean, resolve relative URLs to absolute, and deduplicate
|
|
134
|
+
images.map do |img|
|
|
135
|
+
clean_url = img.gsub(/&/, '&').strip
|
|
136
|
+
resolve_url(clean_url, base)
|
|
137
|
+
end.compact.reject(&:empty?).uniq
|
|
138
|
+
end
|
|
139
|
+
|
|
70
140
|
def [](key)
|
|
71
141
|
if parsed_response.is_a?(Hash) || parsed_response.is_a?(Array)
|
|
72
142
|
parsed_response[key]
|
|
@@ -92,6 +162,20 @@ module HttpMimic
|
|
|
92
162
|
"#<#{self.class.name}:0x#{object_id.to_s(16)} @code=#{code} @status_message=#{status_message.inspect} @headers=#{headers.to_h.inspect} @parsed_response=#{parsed_response.inspect}>"
|
|
93
163
|
end
|
|
94
164
|
|
|
165
|
+
private
|
|
166
|
+
|
|
167
|
+
def resolve_url(url, base)
|
|
168
|
+
return nil if url.nil? || url.empty? || url.start_with?('data:', 'javascript:', 'blob:', '#')
|
|
169
|
+
return url if url =~ /\Ahttps?:\/\//i
|
|
170
|
+
return "https:#{url}" if url.start_with?('//')
|
|
171
|
+
|
|
172
|
+
return url if base.nil? || base.empty?
|
|
173
|
+
require 'uri'
|
|
174
|
+
URI.join(base, url).to_s
|
|
175
|
+
rescue URI::InvalidURIError
|
|
176
|
+
url
|
|
177
|
+
end
|
|
178
|
+
|
|
95
179
|
def method_missing(method_name, *args, &block)
|
|
96
180
|
if parsed_response.respond_to?(method_name)
|
|
97
181
|
parsed_response.public_send(method_name, *args, &block)
|
|
@@ -5,6 +5,8 @@ require 'json'
|
|
|
5
5
|
module HttpMimic
|
|
6
6
|
class ResponseParser
|
|
7
7
|
HTTP_STATUS_LINE_REGEX = /\AHTTP\/(?<version>[\d\.]+)\s+(?<code>\d{3})(?:\s+(?<message>.*))?/i
|
|
8
|
+
HTTP_STATUS_LINE_B = /\AHTTP\/(?:[\d\.]+)\s+\d{3}/i
|
|
9
|
+
BINARY_MIME_KEYWORDS = %w[image/ audio/ video/ pdf octet-stream zip gzip tar compressed stream font wasm].freeze
|
|
8
10
|
|
|
9
11
|
attr_reader :raw_output, :exit_status, :stderr, :command, :request_url
|
|
10
12
|
|
|
@@ -17,8 +19,8 @@ module HttpMimic
|
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
def parse
|
|
20
|
-
# Split header blocks and body
|
|
21
|
-
header_blocks,
|
|
22
|
+
# Split header blocks and body in binary-safe manner
|
|
23
|
+
header_blocks, raw_body = split_headers_and_body(@raw_output)
|
|
22
24
|
|
|
23
25
|
final_header_block = header_blocks.last || ''
|
|
24
26
|
history_header_blocks = header_blocks.size > 1 ? header_blocks[0...-1] : []
|
|
@@ -39,6 +41,14 @@ module HttpMimic
|
|
|
39
41
|
}
|
|
40
42
|
end
|
|
41
43
|
|
|
44
|
+
# Process body: retain raw bytes for binary, or UTF-8 encode for text
|
|
45
|
+
content_type = headers['content-type'].to_s.downcase
|
|
46
|
+
body = if binary_content?(content_type, raw_body)
|
|
47
|
+
raw_body
|
|
48
|
+
else
|
|
49
|
+
raw_body.dup.force_encoding('UTF-8').scrub
|
|
50
|
+
end
|
|
51
|
+
|
|
42
52
|
# Parse body (auto-detect JSON)
|
|
43
53
|
parsed_body = parse_body(body, headers)
|
|
44
54
|
|
|
@@ -61,32 +71,36 @@ module HttpMimic
|
|
|
61
71
|
|
|
62
72
|
private
|
|
63
73
|
|
|
64
|
-
# Split raw_output into header blocks and body
|
|
74
|
+
# Split raw_output into header blocks and body in a binary-safe manner
|
|
65
75
|
def split_headers_and_body(text)
|
|
66
|
-
return [[], ''] if text.nil? || text.empty?
|
|
67
|
-
|
|
68
|
-
# Normalize line endings
|
|
69
|
-
normalized = text.gsub(/\r\n/, "\n")
|
|
70
|
-
parts = normalized.split("\n\n")
|
|
76
|
+
return [[], ''.b] if text.nil? || text.empty?
|
|
71
77
|
|
|
78
|
+
remaining = text.to_s.b
|
|
72
79
|
header_blocks = []
|
|
73
|
-
body_index = 0
|
|
74
80
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
81
|
+
while remaining =~ HTTP_STATUS_LINE_B
|
|
82
|
+
crlf_idx = remaining.index("\r\n\r\n".b)
|
|
83
|
+
lf_idx = remaining.index("\n\n".b)
|
|
84
|
+
|
|
85
|
+
break unless crlf_idx || lf_idx
|
|
86
|
+
|
|
87
|
+
if crlf_idx && lf_idx
|
|
88
|
+
delim_pos = [crlf_idx, lf_idx].min
|
|
89
|
+
delim_len = (delim_pos == crlf_idx) ? 4 : 2
|
|
90
|
+
elsif crlf_idx
|
|
91
|
+
delim_pos = crlf_idx
|
|
92
|
+
delim_len = 4
|
|
80
93
|
else
|
|
81
|
-
|
|
82
|
-
|
|
94
|
+
delim_pos = lf_idx
|
|
95
|
+
delim_len = 2
|
|
83
96
|
end
|
|
84
|
-
end
|
|
85
97
|
|
|
86
|
-
|
|
87
|
-
|
|
98
|
+
header_block = remaining[0...delim_pos].force_encoding('UTF-8').scrub
|
|
99
|
+
header_blocks << header_block
|
|
100
|
+
remaining = remaining[(delim_pos + delim_len)..-1] || ''.b
|
|
101
|
+
end
|
|
88
102
|
|
|
89
|
-
[header_blocks,
|
|
103
|
+
[header_blocks, remaining]
|
|
90
104
|
end
|
|
91
105
|
|
|
92
106
|
def parse_header_block(block)
|
|
@@ -129,12 +143,22 @@ module HttpMimic
|
|
|
129
143
|
[code, http_version, status_message, headers, cookies]
|
|
130
144
|
end
|
|
131
145
|
|
|
146
|
+
def binary_content?(content_type, body)
|
|
147
|
+
return true if BINARY_MIME_KEYWORDS.any? { |kw| content_type.include?(kw) }
|
|
148
|
+
|
|
149
|
+
# Check for null bytes in the initial portion of body
|
|
150
|
+
sample = body[0, 1024]
|
|
151
|
+
sample&.include?("\x00".b)
|
|
152
|
+
end
|
|
153
|
+
|
|
132
154
|
def parse_body(body, headers)
|
|
133
155
|
return nil if body.nil? || body.empty?
|
|
134
156
|
|
|
135
157
|
content_type = headers['content-type'].to_s.downcase
|
|
158
|
+
return body if binary_content?(content_type, body)
|
|
136
159
|
|
|
137
|
-
|
|
160
|
+
trimmed = body.strip
|
|
161
|
+
if content_type.include?('json') || trimmed.start_with?('{', '[')
|
|
138
162
|
begin
|
|
139
163
|
JSON.parse(body)
|
|
140
164
|
rescue JSON::ParserError
|
data/lib/http_mimic/version.rb
CHANGED