ask-web-fetch 0.7.7 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/ask/web_fetch/backends/browser.rb +44 -5
- data/lib/ask/web_fetch/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e03652c72eebca6e1d1fd7b86903a8630936c788b15be62e1b8cc5a43433fac2
|
|
4
|
+
data.tar.gz: 5fa4e325ee99f52d5a7f9a816d2fe66703ad0040f5cbc4e034f3f0924815d13f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a3d9afedbe10bbebdc85eedc1b693c3dcc7e607226c87aaab7e879757de8eb339d4aea2005fc809529f94c9570dc9ff81f3beccac2fdc9656a9d0fdf6cfd7a4e
|
|
7
|
+
data.tar.gz: 65a83c8a2b102ec7b36630c361f56f4828e28ec9a75333e255a339b056752b86312a7becdc7139c1ce9117255261f2ddea4f1fbde0c17f3b488d3c452ce1b3f4
|
|
@@ -54,6 +54,16 @@ module Ask
|
|
|
54
54
|
# How often to poll for the challenge to clear.
|
|
55
55
|
POLL_INTERVAL = 0.5
|
|
56
56
|
|
|
57
|
+
# Brave paths first — Brave's fingerprinting resistance (canvas,
|
|
58
|
+
# WebGL, audio randomization) makes it harder for Cloudflare/
|
|
59
|
+
# DataDome to flag automation than stock Chrome/Chromium.
|
|
60
|
+
BRAVE_PATHS = [
|
|
61
|
+
'/Applications/Brave Browser.app/Contents/MacOS/Brave Browser',
|
|
62
|
+
'/usr/bin/brave-browser',
|
|
63
|
+
'/usr/bin/brave-browser-stable',
|
|
64
|
+
'/opt/brave.com/brave/brave-browser'
|
|
65
|
+
].freeze
|
|
66
|
+
|
|
57
67
|
DEFAULT_PATHS = [
|
|
58
68
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
59
69
|
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
@@ -64,6 +74,11 @@ module Ask
|
|
|
64
74
|
'/opt/google/chrome/chrome'
|
|
65
75
|
].freeze
|
|
66
76
|
|
|
77
|
+
# Random viewport dimensions (FHD to 2K) — a fixed window size is
|
|
78
|
+
# a fingerprinting signal. Each fetch gets a fresh random size.
|
|
79
|
+
VIEWPORT_WIDTH_RANGE = (1920..2560).freeze
|
|
80
|
+
VIEWPORT_HEIGHT_RANGE = (1080..1440).freeze
|
|
81
|
+
|
|
67
82
|
class << self
|
|
68
83
|
# The shared browser (or a test double). Reuse keeps the solved
|
|
69
84
|
# challenge cookie warm across fetches within one process.
|
|
@@ -72,14 +87,17 @@ module Ask
|
|
|
72
87
|
# The ContentFilter applied to every page by default.
|
|
73
88
|
attr_writer :content_filter
|
|
74
89
|
|
|
75
|
-
# Absolute path to a Chrome/Chromium binary; nil when not found.
|
|
90
|
+
# Absolute path to a Chrome/Chromium/Brave binary; nil when not found.
|
|
91
|
+
# Prefers Brave (fingerprinting resistance) over Chrome/Chromium.
|
|
92
|
+
# Override with ASK_WEB_FETCH_BROWSER_PATH (any Chromium-based
|
|
93
|
+
# browser) or the legacy ASK_WEB_FETCH_CHROME_PATH.
|
|
76
94
|
attr_writer :path
|
|
77
95
|
|
|
78
|
-
# CDP endpoint of an already-running
|
|
96
|
+
# CDP endpoint of an already-running browser ("http://host:port",
|
|
79
97
|
# "…/json/version", or a ws:// browser URL) to attach to instead.
|
|
80
98
|
attr_writer :cdp_url
|
|
81
99
|
|
|
82
|
-
attr_writer :challenge_timeout, :poll_interval
|
|
100
|
+
attr_writer :challenge_timeout, :poll_interval, :viewport
|
|
83
101
|
|
|
84
102
|
# Domains already given a warm pass this process — a failed warm
|
|
85
103
|
# (DataDome-class wall) is not re-paid at CHALLENGE_TIMEOUT on
|
|
@@ -107,7 +125,26 @@ module Ask
|
|
|
107
125
|
end
|
|
108
126
|
|
|
109
127
|
def path
|
|
110
|
-
@path || ENV['
|
|
128
|
+
@path || ENV['ASK_WEB_FETCH_BROWSER_PATH'] || ENV['ASK_WEB_FETCH_CHROME_PATH'] ||
|
|
129
|
+
BRAVE_PATHS.find { |p| File.exist?(p) } ||
|
|
130
|
+
DEFAULT_PATHS.find { |p| File.exist?(p) }
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# A fresh random viewport for each browser launch — a fixed
|
|
134
|
+
# window size is a fingerprinting signal. Returns { width:,
|
|
135
|
+
# height: } in the FHD-to-2K range. Override with
|
|
136
|
+
# Browser.viewport = { width: 1920, height: 1080 } for tests.
|
|
137
|
+
def viewport
|
|
138
|
+
@viewport || {
|
|
139
|
+
width: rand(VIEWPORT_WIDTH_RANGE),
|
|
140
|
+
height: rand(VIEWPORT_HEIGHT_RANGE)
|
|
141
|
+
}
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# True when the configured binary is Brave (fingerprinting-
|
|
145
|
+
# resistant) rather than stock Chrome/Chromium.
|
|
146
|
+
def brave?
|
|
147
|
+
path.to_s.match?(/brave/i)
|
|
111
148
|
end
|
|
112
149
|
|
|
113
150
|
def cdp_url
|
|
@@ -152,11 +189,13 @@ module Ask
|
|
|
152
189
|
def build_browser
|
|
153
190
|
return AttachedBrowser.new(ws_url_for(cdp_url), timeout: CHALLENGE_TIMEOUT + IDLE_TIMEOUT) if cdp_url
|
|
154
191
|
|
|
192
|
+
vp = viewport
|
|
155
193
|
Ferrum::Browser.new(
|
|
156
194
|
browser_path: path,
|
|
157
195
|
headless: true,
|
|
158
196
|
user_data_dir: ENV['ASK_WEB_FETCH_PROFILE'],
|
|
159
|
-
timeout: CHALLENGE_TIMEOUT + IDLE_TIMEOUT
|
|
197
|
+
timeout: CHALLENGE_TIMEOUT + IDLE_TIMEOUT,
|
|
198
|
+
window_size: [vp[:width], vp[:height]]
|
|
160
199
|
)
|
|
161
200
|
end
|
|
162
201
|
end
|