kimurai 1.3.2 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +7 -6
- data/lib/kimurai/base.rb +5 -1
- data/lib/kimurai/browser_builder.rb +8 -26
- data/lib/kimurai/browser_builder/mechanize_builder.rb +120 -116
- data/lib/kimurai/browser_builder/poltergeist_phantomjs_builder.rb +139 -135
- data/lib/kimurai/browser_builder/selenium_chrome_builder.rb +152 -148
- data/lib/kimurai/browser_builder/selenium_firefox_builder.rb +161 -157
- data/lib/kimurai/capybara_ext/session.rb +10 -1
- data/lib/kimurai/capybara_ext/session/config.rb +1 -1
- data/lib/kimurai/cli.rb +5 -1
- data/lib/kimurai/template/Gemfile +1 -1
- data/lib/kimurai/template/spiders/application_spider.rb +6 -0
- data/lib/kimurai/version.rb +1 -1
- metadata +2 -2
|
@@ -4,191 +4,195 @@ require_relative '../capybara_configuration'
|
|
|
4
4
|
require_relative '../capybara_ext/selenium/driver'
|
|
5
5
|
require_relative '../capybara_ext/session'
|
|
6
6
|
|
|
7
|
-
module Kimurai
|
|
8
|
-
class
|
|
9
|
-
class
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
end
|
|
7
|
+
module Kimurai::BrowserBuilder
|
|
8
|
+
class SeleniumChromeBuilder
|
|
9
|
+
class << self
|
|
10
|
+
attr_accessor :virtual_display
|
|
11
|
+
end
|
|
13
12
|
|
|
14
|
-
|
|
13
|
+
attr_reader :logger, :spider
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
15
|
+
def initialize(config, spider:)
|
|
16
|
+
@config = config
|
|
17
|
+
@spider = spider
|
|
18
|
+
@logger = spider.logger
|
|
19
|
+
end
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
21
|
+
def build
|
|
22
|
+
# Register driver
|
|
23
|
+
Capybara.register_driver :selenium_chrome do |app|
|
|
24
|
+
# Create driver options
|
|
25
|
+
opts = { args: %w[--disable-gpu --no-sandbox --disable-translate] }
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
# Provide custom chrome browser path:
|
|
28
|
+
if chrome_path = Kimurai.configuration.selenium_chrome_path
|
|
29
|
+
opts.merge!(binary: chrome_path)
|
|
30
|
+
end
|
|
32
31
|
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
# See all options here: https://seleniumhq.github.io/selenium/docs/api/rb/Selenium/WebDriver/Chrome/Options.html
|
|
33
|
+
driver_options = Selenium::WebDriver::Chrome::Options.new(opts)
|
|
35
34
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
35
|
+
# Window size
|
|
36
|
+
if size = @config[:window_size].presence
|
|
37
|
+
driver_options.args << "--window-size=#{size.join(',')}"
|
|
38
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled window_size"
|
|
39
|
+
end
|
|
41
40
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if %w(http socks5).include?(type)
|
|
48
|
-
if user.nil? && password.nil?
|
|
49
|
-
driver_options.args << "--proxy-server=#{type}://#{ip}:#{port}"
|
|
50
|
-
logger.debug "BrowserBuilder (selenium_chrome): enabled #{type} proxy, ip: #{ip}, port: #{port}"
|
|
51
|
-
else
|
|
52
|
-
logger.error "BrowserBuilder (selenium_chrome): proxy with authentication doesn't supported by selenium, skipped"
|
|
53
|
-
end
|
|
54
|
-
else
|
|
55
|
-
logger.error "BrowserBuilder (selenium_chrome): wrong type of proxy: #{type}, skipped"
|
|
56
|
-
end
|
|
57
|
-
end
|
|
41
|
+
# Proxy
|
|
42
|
+
if proxy = @config[:proxy].presence
|
|
43
|
+
proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
|
|
44
|
+
ip, port, type, user, password = proxy_string.split(":")
|
|
58
45
|
|
|
59
|
-
if
|
|
60
|
-
if
|
|
61
|
-
driver_options.args << "--proxy-
|
|
62
|
-
logger.debug "BrowserBuilder (selenium_chrome): enabled
|
|
46
|
+
if %w(http socks5).include?(type)
|
|
47
|
+
if user.nil? && password.nil?
|
|
48
|
+
driver_options.args << "--proxy-server=#{type}://#{ip}:#{port}"
|
|
49
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled #{type} proxy, ip: #{ip}, port: #{port}"
|
|
63
50
|
else
|
|
64
|
-
logger.error "BrowserBuilder (selenium_chrome):
|
|
51
|
+
logger.error "BrowserBuilder (selenium_chrome): proxy with authentication doesn't supported by selenium, skipped"
|
|
65
52
|
end
|
|
53
|
+
else
|
|
54
|
+
logger.error "BrowserBuilder (selenium_chrome): wrong type of proxy: #{type}, skipped"
|
|
66
55
|
end
|
|
56
|
+
end
|
|
67
57
|
|
|
68
|
-
|
|
69
|
-
if
|
|
70
|
-
driver_options.args << "--
|
|
71
|
-
|
|
72
|
-
|
|
58
|
+
if proxy_bypass_list = @config[:proxy_bypass_list].presence
|
|
59
|
+
if proxy
|
|
60
|
+
driver_options.args << "--proxy-bypass-list=#{proxy_bypass_list.join(';')}"
|
|
61
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled proxy_bypass_list"
|
|
62
|
+
else
|
|
63
|
+
logger.error "BrowserBuilder (selenium_chrome): provide `proxy` to set proxy_bypass_list, skipped"
|
|
73
64
|
end
|
|
65
|
+
end
|
|
74
66
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
67
|
+
# SSL
|
|
68
|
+
if @config[:ignore_ssl_errors].present?
|
|
69
|
+
driver_options.args << "--ignore-certificate-errors"
|
|
70
|
+
driver_options.args << "--allow-insecure-localhost"
|
|
71
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled ignore_ssl_errors"
|
|
72
|
+
end
|
|
80
73
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
74
|
+
# Disable images
|
|
75
|
+
if @config[:disable_images].present?
|
|
76
|
+
driver_options.prefs["profile.managed_default_content_settings.images"] = 2
|
|
77
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled disable_images"
|
|
78
|
+
end
|
|
85
79
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
end
|
|
80
|
+
# Headers
|
|
81
|
+
if @config[:headers].present?
|
|
82
|
+
logger.warn "BrowserBuilder: (selenium_chrome): custom headers doesn't supported by selenium, skipped"
|
|
83
|
+
end
|
|
91
84
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
85
|
+
if user_agent = @config[:user_agent].presence
|
|
86
|
+
user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
|
|
87
|
+
driver_options.args << "--user-agent='#{user_agent_string}'"
|
|
88
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled custom user_agent"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Headless mode
|
|
92
|
+
if ENV["HEADLESS"] != "false"
|
|
93
|
+
if @config[:headless_mode] == :virtual_display
|
|
94
|
+
if Gem::Platform.local.os == "linux"
|
|
95
|
+
unless self.class.virtual_display
|
|
96
|
+
require 'headless'
|
|
97
|
+
self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
|
|
98
|
+
self.class.virtual_display.start
|
|
106
99
|
end
|
|
100
|
+
|
|
101
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled virtual_display headless_mode"
|
|
107
102
|
else
|
|
108
|
-
|
|
109
|
-
|
|
103
|
+
logger.error "BrowserBuilder (selenium_chrome): virtual_display headless_mode works only " \
|
|
104
|
+
"on Linux platform. Browser will run in normal mode. Set `native` mode instead."
|
|
110
105
|
end
|
|
106
|
+
else
|
|
107
|
+
driver_options.args << "--headless"
|
|
108
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled native headless_mode"
|
|
111
109
|
end
|
|
112
|
-
|
|
113
|
-
chromedriver_path = Kimurai.configuration.chromedriver_path || "/usr/local/bin/chromedriver"
|
|
114
|
-
Capybara::Selenium::Driver.new(app, browser: :chrome, options: driver_options, driver_path: chromedriver_path)
|
|
115
110
|
end
|
|
116
111
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
logger.debug "BrowserBuilder (selenium_chrome): created browser instance"
|
|
112
|
+
chromedriver_path = Kimurai.configuration.chromedriver_path || "/usr/local/bin/chromedriver"
|
|
113
|
+
Capybara::Selenium::Driver.new(app, browser: :chrome, options: driver_options, driver_path: chromedriver_path)
|
|
114
|
+
end
|
|
121
115
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
116
|
+
# Create browser instance (Capybara session)
|
|
117
|
+
@browser = Capybara::Session.new(:selenium_chrome)
|
|
118
|
+
@browser.spider = spider
|
|
119
|
+
logger.debug "BrowserBuilder (selenium_chrome): created browser instance"
|
|
125
120
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
logger.debug "BrowserBuilder (selenium_chrome): enabled custom cookies"
|
|
130
|
-
end
|
|
121
|
+
if @config[:extensions].present?
|
|
122
|
+
logger.error "BrowserBuilder (selenium_chrome): `extensions` option not supported by Selenium, skipped"
|
|
123
|
+
end
|
|
131
124
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
end
|
|
125
|
+
# Cookies
|
|
126
|
+
if cookies = @config[:cookies].presence
|
|
127
|
+
@browser.config.cookies = cookies
|
|
128
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled custom cookies"
|
|
129
|
+
end
|
|
138
130
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
131
|
+
# Browser instance options
|
|
132
|
+
# skip_request_errors
|
|
133
|
+
if skip_errors = @config[:skip_request_errors].presence
|
|
134
|
+
@browser.config.skip_request_errors = skip_errors
|
|
135
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled skip_request_errors"
|
|
136
|
+
end
|
|
144
137
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
138
|
+
# retry_request_errors
|
|
139
|
+
if retry_errors = @config[:retry_request_errors].presence
|
|
140
|
+
@browser.config.retry_request_errors = retry_errors
|
|
141
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled retry_request_errors"
|
|
142
|
+
end
|
|
150
143
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
144
|
+
# restart_if
|
|
145
|
+
if requests_limit = @config.dig(:restart_if, :requests_limit).presence
|
|
146
|
+
@browser.config.restart_if[:requests_limit] = requests_limit
|
|
147
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.requests_limit >= #{requests_limit}"
|
|
148
|
+
end
|
|
155
149
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
end
|
|
150
|
+
if memory_limit = @config.dig(:restart_if, :memory_limit).presence
|
|
151
|
+
@browser.config.restart_if[:memory_limit] = memory_limit
|
|
152
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.memory_limit >= #{memory_limit}"
|
|
153
|
+
end
|
|
161
154
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_and_set_cookies"
|
|
168
|
-
else
|
|
169
|
-
logger.error "BrowserBuilder (selenium_chrome): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
|
|
170
|
-
end
|
|
171
|
-
end
|
|
155
|
+
# before_request clear_cookies
|
|
156
|
+
if @config.dig(:before_request, :clear_cookies)
|
|
157
|
+
@browser.config.before_request[:clear_cookies] = true
|
|
158
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_cookies"
|
|
159
|
+
end
|
|
172
160
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
161
|
+
# before_request clear_and_set_cookies
|
|
162
|
+
if @config.dig(:before_request, :clear_and_set_cookies)
|
|
163
|
+
if cookies = @config[:cookies].presence
|
|
164
|
+
@browser.config.cookies = cookies
|
|
165
|
+
@browser.config.before_request[:clear_and_set_cookies] = true
|
|
166
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_and_set_cookies"
|
|
167
|
+
else
|
|
168
|
+
logger.error "BrowserBuilder (selenium_chrome): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
|
|
176
169
|
end
|
|
170
|
+
end
|
|
177
171
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
172
|
+
# before_request change_user_agent
|
|
173
|
+
if @config.dig(:before_request, :change_user_agent)
|
|
174
|
+
logger.error "BrowserBuilder (selenium_chrome): before_request.change_user_agent option not supported by Selenium, skipped"
|
|
175
|
+
end
|
|
182
176
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
end
|
|
177
|
+
# before_request change_proxy
|
|
178
|
+
if @config.dig(:before_request, :change_proxy)
|
|
179
|
+
logger.error "BrowserBuilder (selenium_chrome): before_request.change_proxy option not supported by Selenium, skipped"
|
|
180
|
+
end
|
|
188
181
|
|
|
189
|
-
|
|
190
|
-
|
|
182
|
+
# before_request delay
|
|
183
|
+
if delay = @config.dig(:before_request, :delay).presence
|
|
184
|
+
@browser.config.before_request[:delay] = delay
|
|
185
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.delay"
|
|
191
186
|
end
|
|
187
|
+
|
|
188
|
+
# encoding
|
|
189
|
+
if encoding = @config[:encoding]
|
|
190
|
+
@browser.config.encoding = encoding
|
|
191
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled encoding: #{encoding}"
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# return Capybara session instance
|
|
195
|
+
@browser
|
|
192
196
|
end
|
|
193
197
|
end
|
|
194
198
|
end
|
|
@@ -4,197 +4,201 @@ require_relative '../capybara_configuration'
|
|
|
4
4
|
require_relative '../capybara_ext/selenium/driver'
|
|
5
5
|
require_relative '../capybara_ext/session'
|
|
6
6
|
|
|
7
|
-
module Kimurai
|
|
8
|
-
class
|
|
9
|
-
class
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
end
|
|
7
|
+
module Kimurai::BrowserBuilder
|
|
8
|
+
class SeleniumFirefoxBuilder
|
|
9
|
+
class << self
|
|
10
|
+
attr_accessor :virtual_display
|
|
11
|
+
end
|
|
13
12
|
|
|
14
|
-
|
|
13
|
+
attr_reader :logger, :spider
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
15
|
+
def initialize(config, spider:)
|
|
16
|
+
@config = config
|
|
17
|
+
@spider = spider
|
|
18
|
+
@logger = spider.logger
|
|
19
|
+
end
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
else
|
|
53
|
-
logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
|
|
54
|
-
end
|
|
21
|
+
def build
|
|
22
|
+
# Register driver
|
|
23
|
+
Capybara.register_driver :selenium_firefox do |app|
|
|
24
|
+
# Create driver options
|
|
25
|
+
driver_options = Selenium::WebDriver::Firefox::Options.new
|
|
26
|
+
driver_options.profile = Selenium::WebDriver::Firefox::Profile.new
|
|
27
|
+
driver_options.profile["browser.link.open_newwindow"] = 3 # open windows in tabs
|
|
28
|
+
driver_options.profile["media.peerconnection.enabled"] = false # disable web rtc
|
|
29
|
+
|
|
30
|
+
# Proxy
|
|
31
|
+
if proxy = @config[:proxy].presence
|
|
32
|
+
proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
|
|
33
|
+
ip, port, type, user, password = proxy_string.split(":")
|
|
34
|
+
|
|
35
|
+
if user.nil? && password.nil?
|
|
36
|
+
driver_options.profile["network.proxy.type"] = 1
|
|
37
|
+
if type == "http"
|
|
38
|
+
driver_options.profile["network.proxy.http"] = ip
|
|
39
|
+
driver_options.profile["network.proxy.http_port"] = port.to_i
|
|
40
|
+
driver_options.profile["network.proxy.ssl"] = ip
|
|
41
|
+
driver_options.profile["network.proxy.ssl_port"] = port.to_i
|
|
42
|
+
|
|
43
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled http proxy, ip: #{ip}, port: #{port}"
|
|
44
|
+
elsif type == "socks5"
|
|
45
|
+
driver_options.profile["network.proxy.socks"] = ip
|
|
46
|
+
driver_options.profile["network.proxy.socks_port"] = port.to_i
|
|
47
|
+
driver_options.profile["network.proxy.socks_version"] = 5
|
|
48
|
+
driver_options.profile["network.proxy.socks_remote_dns"] = true
|
|
49
|
+
|
|
50
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled socks5 proxy, ip: #{ip}, port: #{port}"
|
|
55
51
|
else
|
|
56
|
-
logger.error "BrowserBuilder (selenium_firefox):
|
|
52
|
+
logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
|
|
57
53
|
end
|
|
54
|
+
else
|
|
55
|
+
logger.error "BrowserBuilder (selenium_firefox): proxy with authentication doesn't supported by selenium, skipped"
|
|
58
56
|
end
|
|
57
|
+
end
|
|
59
58
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
end
|
|
59
|
+
if proxy_bypass_list = @config[:proxy_bypass_list].presence
|
|
60
|
+
if proxy
|
|
61
|
+
driver_options.profile["network.proxy.no_proxies_on"] = proxy_bypass_list.join(", ")
|
|
62
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled proxy_bypass_list"
|
|
63
|
+
else
|
|
64
|
+
logger.error "BrowserBuilder (selenium_firefox): provide `proxy` to set proxy_bypass_list, skipped"
|
|
67
65
|
end
|
|
66
|
+
end
|
|
68
67
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
68
|
+
# SSL
|
|
69
|
+
if @config[:ignore_ssl_errors].present?
|
|
70
|
+
driver_options.profile.secure_ssl = false
|
|
71
|
+
driver_options.profile.assume_untrusted_certificate_issuer = true
|
|
72
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled ignore_ssl_errors"
|
|
73
|
+
end
|
|
75
74
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
75
|
+
# Disable images
|
|
76
|
+
if @config[:disable_images].present?
|
|
77
|
+
driver_options.profile["permissions.default.image"] = 2
|
|
78
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled disable_images"
|
|
79
|
+
end
|
|
81
80
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
81
|
+
# Headers
|
|
82
|
+
if @config[:headers].present?
|
|
83
|
+
logger.warn "BrowserBuilder: (selenium_firefox): custom headers doesn't supported by selenium, skipped"
|
|
84
|
+
end
|
|
86
85
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
86
|
+
if user_agent = @config[:user_agent].presence
|
|
87
|
+
user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
|
|
88
|
+
driver_options.profile["general.useragent.override"] = user_agent_string
|
|
89
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled custom user_agent"
|
|
90
|
+
end
|
|
92
91
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
|
|
104
|
-
else
|
|
105
|
-
logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
|
|
106
|
-
"on Linux platform. Browser will run in normal mode. Set `native` mode instead."
|
|
92
|
+
# Headless mode
|
|
93
|
+
if ENV["HEADLESS"] != "false"
|
|
94
|
+
if @config[:headless_mode] == :virtual_display
|
|
95
|
+
if Gem::Platform.local.os == "linux"
|
|
96
|
+
unless self.class.virtual_display
|
|
97
|
+
require 'headless'
|
|
98
|
+
self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
|
|
99
|
+
self.class.virtual_display.start
|
|
107
100
|
end
|
|
101
|
+
|
|
102
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
|
|
108
103
|
else
|
|
109
|
-
|
|
110
|
-
|
|
104
|
+
logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
|
|
105
|
+
"on Linux platform. Browser will run in normal mode. Set `native` mode instead."
|
|
111
106
|
end
|
|
107
|
+
else
|
|
108
|
+
driver_options.args << "--headless"
|
|
109
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled native headless_mode"
|
|
112
110
|
end
|
|
113
|
-
|
|
114
|
-
Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
|
|
115
111
|
end
|
|
116
112
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
@browser.spider = spider
|
|
120
|
-
logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
|
|
113
|
+
Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
|
|
114
|
+
end
|
|
121
115
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
116
|
+
# Create browser instance (Capybara session)
|
|
117
|
+
@browser = Capybara::Session.new(:selenium_firefox)
|
|
118
|
+
@browser.spider = spider
|
|
119
|
+
logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
|
|
125
120
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
|
|
130
|
-
end
|
|
121
|
+
if @config[:extensions].present?
|
|
122
|
+
logger.error "BrowserBuilder (selenium_firefox): `extensions` option not supported by Selenium, skipped"
|
|
123
|
+
end
|
|
131
124
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
125
|
+
# Window size
|
|
126
|
+
if size = @config[:window_size].presence
|
|
127
|
+
@browser.current_window.resize_to(*size)
|
|
128
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
|
|
129
|
+
end
|
|
137
130
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
end
|
|
131
|
+
# Cookies
|
|
132
|
+
if cookies = @config[:cookies].presence
|
|
133
|
+
@browser.config.cookies = cookies
|
|
134
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled custom cookies"
|
|
135
|
+
end
|
|
144
136
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
137
|
+
# Browser instance options
|
|
138
|
+
# skip_request_errors
|
|
139
|
+
if skip_errors = @config[:skip_request_errors].presence
|
|
140
|
+
@browser.config.skip_request_errors = skip_errors
|
|
141
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled skip_request_errors"
|
|
142
|
+
end
|
|
150
143
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
144
|
+
# retry_request_errors
|
|
145
|
+
if retry_errors = @config[:retry_request_errors].presence
|
|
146
|
+
@browser.config.retry_request_errors = retry_errors
|
|
147
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled retry_request_errors"
|
|
148
|
+
end
|
|
156
149
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
150
|
+
# restart_if
|
|
151
|
+
if requests_limit = @config.dig(:restart_if, :requests_limit).presence
|
|
152
|
+
@browser.config.restart_if[:requests_limit] = requests_limit
|
|
153
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.requests_limit >= #{requests_limit}"
|
|
154
|
+
end
|
|
161
155
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
end
|
|
156
|
+
if memory_limit = @config.dig(:restart_if, :memory_limit).presence
|
|
157
|
+
@browser.config.restart_if[:memory_limit] = memory_limit
|
|
158
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.memory_limit >= #{memory_limit}"
|
|
159
|
+
end
|
|
167
160
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
|
|
174
|
-
else
|
|
175
|
-
logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
|
|
176
|
-
end
|
|
177
|
-
end
|
|
161
|
+
# before_request clear_cookies
|
|
162
|
+
if @config.dig(:before_request, :clear_cookies)
|
|
163
|
+
@browser.config.before_request[:clear_cookies] = true
|
|
164
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_cookies"
|
|
165
|
+
end
|
|
178
166
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
167
|
+
# before_request clear_and_set_cookies
|
|
168
|
+
if @config.dig(:before_request, :clear_and_set_cookies)
|
|
169
|
+
if cookies = @config[:cookies].presence
|
|
170
|
+
@browser.config.cookies = cookies
|
|
171
|
+
@browser.config.before_request[:clear_and_set_cookies] = true
|
|
172
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
|
|
173
|
+
else
|
|
174
|
+
logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
|
|
182
175
|
end
|
|
176
|
+
end
|
|
183
177
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
178
|
+
# before_request change_user_agent
|
|
179
|
+
if @config.dig(:before_request, :change_user_agent)
|
|
180
|
+
logger.error "BrowserBuilder (selenium_firefox): before_request.change_user_agent option not supported by Selenium, skipped"
|
|
181
|
+
end
|
|
188
182
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
end
|
|
183
|
+
# before_request change_proxy
|
|
184
|
+
if @config.dig(:before_request, :change_proxy)
|
|
185
|
+
logger.error "BrowserBuilder (selenium_firefox): before_request.change_proxy option not supported by Selenium, skipped"
|
|
186
|
+
end
|
|
194
187
|
|
|
195
|
-
|
|
196
|
-
|
|
188
|
+
# before_request delay
|
|
189
|
+
if delay = @config.dig(:before_request, :delay).presence
|
|
190
|
+
@browser.config.before_request[:delay] = delay
|
|
191
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.delay"
|
|
197
192
|
end
|
|
193
|
+
|
|
194
|
+
# encoding
|
|
195
|
+
if encoding = @config[:encoding]
|
|
196
|
+
@browser.config.encoding = encoding
|
|
197
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled encoding: #{encoding}"
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# return Capybara session instance
|
|
201
|
+
@browser
|
|
198
202
|
end
|
|
199
203
|
end
|
|
200
204
|
end
|