kimurai 1.3.2 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +7 -6
- data/lib/kimurai/base.rb +5 -1
- data/lib/kimurai/browser_builder.rb +8 -26
- data/lib/kimurai/browser_builder/mechanize_builder.rb +120 -116
- data/lib/kimurai/browser_builder/poltergeist_phantomjs_builder.rb +139 -135
- data/lib/kimurai/browser_builder/selenium_chrome_builder.rb +152 -148
- data/lib/kimurai/browser_builder/selenium_firefox_builder.rb +161 -157
- data/lib/kimurai/capybara_ext/session.rb +10 -1
- data/lib/kimurai/capybara_ext/session/config.rb +1 -1
- data/lib/kimurai/cli.rb +5 -1
- data/lib/kimurai/template/Gemfile +1 -1
- data/lib/kimurai/template/spiders/application_spider.rb +6 -0
- data/lib/kimurai/version.rb +1 -1
- metadata +2 -2
|
@@ -4,191 +4,195 @@ require_relative '../capybara_configuration'
|
|
|
4
4
|
require_relative '../capybara_ext/selenium/driver'
|
|
5
5
|
require_relative '../capybara_ext/session'
|
|
6
6
|
|
|
7
|
-
module Kimurai
|
|
8
|
-
class
|
|
9
|
-
class
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
end
|
|
7
|
+
module Kimurai::BrowserBuilder
|
|
8
|
+
class SeleniumChromeBuilder
|
|
9
|
+
class << self
|
|
10
|
+
attr_accessor :virtual_display
|
|
11
|
+
end
|
|
13
12
|
|
|
14
|
-
|
|
13
|
+
attr_reader :logger, :spider
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
15
|
+
def initialize(config, spider:)
|
|
16
|
+
@config = config
|
|
17
|
+
@spider = spider
|
|
18
|
+
@logger = spider.logger
|
|
19
|
+
end
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
21
|
+
def build
|
|
22
|
+
# Register driver
|
|
23
|
+
Capybara.register_driver :selenium_chrome do |app|
|
|
24
|
+
# Create driver options
|
|
25
|
+
opts = { args: %w[--disable-gpu --no-sandbox --disable-translate] }
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
# Provide custom chrome browser path:
|
|
28
|
+
if chrome_path = Kimurai.configuration.selenium_chrome_path
|
|
29
|
+
opts.merge!(binary: chrome_path)
|
|
30
|
+
end
|
|
32
31
|
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
# See all options here: https://seleniumhq.github.io/selenium/docs/api/rb/Selenium/WebDriver/Chrome/Options.html
|
|
33
|
+
driver_options = Selenium::WebDriver::Chrome::Options.new(opts)
|
|
35
34
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
35
|
+
# Window size
|
|
36
|
+
if size = @config[:window_size].presence
|
|
37
|
+
driver_options.args << "--window-size=#{size.join(',')}"
|
|
38
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled window_size"
|
|
39
|
+
end
|
|
41
40
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if %w(http socks5).include?(type)
|
|
48
|
-
if user.nil? && password.nil?
|
|
49
|
-
driver_options.args << "--proxy-server=#{type}://#{ip}:#{port}"
|
|
50
|
-
logger.debug "BrowserBuilder (selenium_chrome): enabled #{type} proxy, ip: #{ip}, port: #{port}"
|
|
51
|
-
else
|
|
52
|
-
logger.error "BrowserBuilder (selenium_chrome): proxy with authentication doesn't supported by selenium, skipped"
|
|
53
|
-
end
|
|
54
|
-
else
|
|
55
|
-
logger.error "BrowserBuilder (selenium_chrome): wrong type of proxy: #{type}, skipped"
|
|
56
|
-
end
|
|
57
|
-
end
|
|
41
|
+
# Proxy
|
|
42
|
+
if proxy = @config[:proxy].presence
|
|
43
|
+
proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
|
|
44
|
+
ip, port, type, user, password = proxy_string.split(":")
|
|
58
45
|
|
|
59
|
-
if
|
|
60
|
-
if
|
|
61
|
-
driver_options.args << "--proxy-
|
|
62
|
-
logger.debug "BrowserBuilder (selenium_chrome): enabled
|
|
46
|
+
if %w(http socks5).include?(type)
|
|
47
|
+
if user.nil? && password.nil?
|
|
48
|
+
driver_options.args << "--proxy-server=#{type}://#{ip}:#{port}"
|
|
49
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled #{type} proxy, ip: #{ip}, port: #{port}"
|
|
63
50
|
else
|
|
64
|
-
logger.error "BrowserBuilder (selenium_chrome):
|
|
51
|
+
logger.error "BrowserBuilder (selenium_chrome): proxy with authentication doesn't supported by selenium, skipped"
|
|
65
52
|
end
|
|
53
|
+
else
|
|
54
|
+
logger.error "BrowserBuilder (selenium_chrome): wrong type of proxy: #{type}, skipped"
|
|
66
55
|
end
|
|
56
|
+
end
|
|
67
57
|
|
|
68
|
-
|
|
69
|
-
if
|
|
70
|
-
driver_options.args << "--
|
|
71
|
-
|
|
72
|
-
|
|
58
|
+
if proxy_bypass_list = @config[:proxy_bypass_list].presence
|
|
59
|
+
if proxy
|
|
60
|
+
driver_options.args << "--proxy-bypass-list=#{proxy_bypass_list.join(';')}"
|
|
61
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled proxy_bypass_list"
|
|
62
|
+
else
|
|
63
|
+
logger.error "BrowserBuilder (selenium_chrome): provide `proxy` to set proxy_bypass_list, skipped"
|
|
73
64
|
end
|
|
65
|
+
end
|
|
74
66
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
67
|
+
# SSL
|
|
68
|
+
if @config[:ignore_ssl_errors].present?
|
|
69
|
+
driver_options.args << "--ignore-certificate-errors"
|
|
70
|
+
driver_options.args << "--allow-insecure-localhost"
|
|
71
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled ignore_ssl_errors"
|
|
72
|
+
end
|
|
80
73
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
74
|
+
# Disable images
|
|
75
|
+
if @config[:disable_images].present?
|
|
76
|
+
driver_options.prefs["profile.managed_default_content_settings.images"] = 2
|
|
77
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled disable_images"
|
|
78
|
+
end
|
|
85
79
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
end
|
|
80
|
+
# Headers
|
|
81
|
+
if @config[:headers].present?
|
|
82
|
+
logger.warn "BrowserBuilder: (selenium_chrome): custom headers doesn't supported by selenium, skipped"
|
|
83
|
+
end
|
|
91
84
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
85
|
+
if user_agent = @config[:user_agent].presence
|
|
86
|
+
user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
|
|
87
|
+
driver_options.args << "--user-agent='#{user_agent_string}'"
|
|
88
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled custom user_agent"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Headless mode
|
|
92
|
+
if ENV["HEADLESS"] != "false"
|
|
93
|
+
if @config[:headless_mode] == :virtual_display
|
|
94
|
+
if Gem::Platform.local.os == "linux"
|
|
95
|
+
unless self.class.virtual_display
|
|
96
|
+
require 'headless'
|
|
97
|
+
self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
|
|
98
|
+
self.class.virtual_display.start
|
|
106
99
|
end
|
|
100
|
+
|
|
101
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled virtual_display headless_mode"
|
|
107
102
|
else
|
|
108
|
-
|
|
109
|
-
|
|
103
|
+
logger.error "BrowserBuilder (selenium_chrome): virtual_display headless_mode works only " \
|
|
104
|
+
"on Linux platform. Browser will run in normal mode. Set `native` mode instead."
|
|
110
105
|
end
|
|
106
|
+
else
|
|
107
|
+
driver_options.args << "--headless"
|
|
108
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled native headless_mode"
|
|
111
109
|
end
|
|
112
|
-
|
|
113
|
-
chromedriver_path = Kimurai.configuration.chromedriver_path || "/usr/local/bin/chromedriver"
|
|
114
|
-
Capybara::Selenium::Driver.new(app, browser: :chrome, options: driver_options, driver_path: chromedriver_path)
|
|
115
110
|
end
|
|
116
111
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
logger.debug "BrowserBuilder (selenium_chrome): created browser instance"
|
|
112
|
+
chromedriver_path = Kimurai.configuration.chromedriver_path || "/usr/local/bin/chromedriver"
|
|
113
|
+
Capybara::Selenium::Driver.new(app, browser: :chrome, options: driver_options, driver_path: chromedriver_path)
|
|
114
|
+
end
|
|
121
115
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
116
|
+
# Create browser instance (Capybara session)
|
|
117
|
+
@browser = Capybara::Session.new(:selenium_chrome)
|
|
118
|
+
@browser.spider = spider
|
|
119
|
+
logger.debug "BrowserBuilder (selenium_chrome): created browser instance"
|
|
125
120
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
logger.debug "BrowserBuilder (selenium_chrome): enabled custom cookies"
|
|
130
|
-
end
|
|
121
|
+
if @config[:extensions].present?
|
|
122
|
+
logger.error "BrowserBuilder (selenium_chrome): `extensions` option not supported by Selenium, skipped"
|
|
123
|
+
end
|
|
131
124
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
end
|
|
125
|
+
# Cookies
|
|
126
|
+
if cookies = @config[:cookies].presence
|
|
127
|
+
@browser.config.cookies = cookies
|
|
128
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled custom cookies"
|
|
129
|
+
end
|
|
138
130
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
131
|
+
# Browser instance options
|
|
132
|
+
# skip_request_errors
|
|
133
|
+
if skip_errors = @config[:skip_request_errors].presence
|
|
134
|
+
@browser.config.skip_request_errors = skip_errors
|
|
135
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled skip_request_errors"
|
|
136
|
+
end
|
|
144
137
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
138
|
+
# retry_request_errors
|
|
139
|
+
if retry_errors = @config[:retry_request_errors].presence
|
|
140
|
+
@browser.config.retry_request_errors = retry_errors
|
|
141
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled retry_request_errors"
|
|
142
|
+
end
|
|
150
143
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
144
|
+
# restart_if
|
|
145
|
+
if requests_limit = @config.dig(:restart_if, :requests_limit).presence
|
|
146
|
+
@browser.config.restart_if[:requests_limit] = requests_limit
|
|
147
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.requests_limit >= #{requests_limit}"
|
|
148
|
+
end
|
|
155
149
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
end
|
|
150
|
+
if memory_limit = @config.dig(:restart_if, :memory_limit).presence
|
|
151
|
+
@browser.config.restart_if[:memory_limit] = memory_limit
|
|
152
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.memory_limit >= #{memory_limit}"
|
|
153
|
+
end
|
|
161
154
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_and_set_cookies"
|
|
168
|
-
else
|
|
169
|
-
logger.error "BrowserBuilder (selenium_chrome): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
|
|
170
|
-
end
|
|
171
|
-
end
|
|
155
|
+
# before_request clear_cookies
|
|
156
|
+
if @config.dig(:before_request, :clear_cookies)
|
|
157
|
+
@browser.config.before_request[:clear_cookies] = true
|
|
158
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_cookies"
|
|
159
|
+
end
|
|
172
160
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
161
|
+
# before_request clear_and_set_cookies
|
|
162
|
+
if @config.dig(:before_request, :clear_and_set_cookies)
|
|
163
|
+
if cookies = @config[:cookies].presence
|
|
164
|
+
@browser.config.cookies = cookies
|
|
165
|
+
@browser.config.before_request[:clear_and_set_cookies] = true
|
|
166
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_and_set_cookies"
|
|
167
|
+
else
|
|
168
|
+
logger.error "BrowserBuilder (selenium_chrome): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
|
|
176
169
|
end
|
|
170
|
+
end
|
|
177
171
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
172
|
+
# before_request change_user_agent
|
|
173
|
+
if @config.dig(:before_request, :change_user_agent)
|
|
174
|
+
logger.error "BrowserBuilder (selenium_chrome): before_request.change_user_agent option not supported by Selenium, skipped"
|
|
175
|
+
end
|
|
182
176
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
end
|
|
177
|
+
# before_request change_proxy
|
|
178
|
+
if @config.dig(:before_request, :change_proxy)
|
|
179
|
+
logger.error "BrowserBuilder (selenium_chrome): before_request.change_proxy option not supported by Selenium, skipped"
|
|
180
|
+
end
|
|
188
181
|
|
|
189
|
-
|
|
190
|
-
|
|
182
|
+
# before_request delay
|
|
183
|
+
if delay = @config.dig(:before_request, :delay).presence
|
|
184
|
+
@browser.config.before_request[:delay] = delay
|
|
185
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.delay"
|
|
191
186
|
end
|
|
187
|
+
|
|
188
|
+
# encoding
|
|
189
|
+
if encoding = @config[:encoding]
|
|
190
|
+
@browser.config.encoding = encoding
|
|
191
|
+
logger.debug "BrowserBuilder (selenium_chrome): enabled encoding: #{encoding}"
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# return Capybara session instance
|
|
195
|
+
@browser
|
|
192
196
|
end
|
|
193
197
|
end
|
|
194
198
|
end
|
|
@@ -4,197 +4,201 @@ require_relative '../capybara_configuration'
|
|
|
4
4
|
require_relative '../capybara_ext/selenium/driver'
|
|
5
5
|
require_relative '../capybara_ext/session'
|
|
6
6
|
|
|
7
|
-
module Kimurai
|
|
8
|
-
class
|
|
9
|
-
class
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
end
|
|
7
|
+
module Kimurai::BrowserBuilder
|
|
8
|
+
class SeleniumFirefoxBuilder
|
|
9
|
+
class << self
|
|
10
|
+
attr_accessor :virtual_display
|
|
11
|
+
end
|
|
13
12
|
|
|
14
|
-
|
|
13
|
+
attr_reader :logger, :spider
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
15
|
+
def initialize(config, spider:)
|
|
16
|
+
@config = config
|
|
17
|
+
@spider = spider
|
|
18
|
+
@logger = spider.logger
|
|
19
|
+
end
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
else
|
|
53
|
-
logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
|
|
54
|
-
end
|
|
21
|
+
def build
|
|
22
|
+
# Register driver
|
|
23
|
+
Capybara.register_driver :selenium_firefox do |app|
|
|
24
|
+
# Create driver options
|
|
25
|
+
driver_options = Selenium::WebDriver::Firefox::Options.new
|
|
26
|
+
driver_options.profile = Selenium::WebDriver::Firefox::Profile.new
|
|
27
|
+
driver_options.profile["browser.link.open_newwindow"] = 3 # open windows in tabs
|
|
28
|
+
driver_options.profile["media.peerconnection.enabled"] = false # disable web rtc
|
|
29
|
+
|
|
30
|
+
# Proxy
|
|
31
|
+
if proxy = @config[:proxy].presence
|
|
32
|
+
proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
|
|
33
|
+
ip, port, type, user, password = proxy_string.split(":")
|
|
34
|
+
|
|
35
|
+
if user.nil? && password.nil?
|
|
36
|
+
driver_options.profile["network.proxy.type"] = 1
|
|
37
|
+
if type == "http"
|
|
38
|
+
driver_options.profile["network.proxy.http"] = ip
|
|
39
|
+
driver_options.profile["network.proxy.http_port"] = port.to_i
|
|
40
|
+
driver_options.profile["network.proxy.ssl"] = ip
|
|
41
|
+
driver_options.profile["network.proxy.ssl_port"] = port.to_i
|
|
42
|
+
|
|
43
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled http proxy, ip: #{ip}, port: #{port}"
|
|
44
|
+
elsif type == "socks5"
|
|
45
|
+
driver_options.profile["network.proxy.socks"] = ip
|
|
46
|
+
driver_options.profile["network.proxy.socks_port"] = port.to_i
|
|
47
|
+
driver_options.profile["network.proxy.socks_version"] = 5
|
|
48
|
+
driver_options.profile["network.proxy.socks_remote_dns"] = true
|
|
49
|
+
|
|
50
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled socks5 proxy, ip: #{ip}, port: #{port}"
|
|
55
51
|
else
|
|
56
|
-
logger.error "BrowserBuilder (selenium_firefox):
|
|
52
|
+
logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
|
|
57
53
|
end
|
|
54
|
+
else
|
|
55
|
+
logger.error "BrowserBuilder (selenium_firefox): proxy with authentication doesn't supported by selenium, skipped"
|
|
58
56
|
end
|
|
57
|
+
end
|
|
59
58
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
end
|
|
59
|
+
if proxy_bypass_list = @config[:proxy_bypass_list].presence
|
|
60
|
+
if proxy
|
|
61
|
+
driver_options.profile["network.proxy.no_proxies_on"] = proxy_bypass_list.join(", ")
|
|
62
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled proxy_bypass_list"
|
|
63
|
+
else
|
|
64
|
+
logger.error "BrowserBuilder (selenium_firefox): provide `proxy` to set proxy_bypass_list, skipped"
|
|
67
65
|
end
|
|
66
|
+
end
|
|
68
67
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
68
|
+
# SSL
|
|
69
|
+
if @config[:ignore_ssl_errors].present?
|
|
70
|
+
driver_options.profile.secure_ssl = false
|
|
71
|
+
driver_options.profile.assume_untrusted_certificate_issuer = true
|
|
72
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled ignore_ssl_errors"
|
|
73
|
+
end
|
|
75
74
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
75
|
+
# Disable images
|
|
76
|
+
if @config[:disable_images].present?
|
|
77
|
+
driver_options.profile["permissions.default.image"] = 2
|
|
78
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled disable_images"
|
|
79
|
+
end
|
|
81
80
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
81
|
+
# Headers
|
|
82
|
+
if @config[:headers].present?
|
|
83
|
+
logger.warn "BrowserBuilder: (selenium_firefox): custom headers doesn't supported by selenium, skipped"
|
|
84
|
+
end
|
|
86
85
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
86
|
+
if user_agent = @config[:user_agent].presence
|
|
87
|
+
user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
|
|
88
|
+
driver_options.profile["general.useragent.override"] = user_agent_string
|
|
89
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled custom user_agent"
|
|
90
|
+
end
|
|
92
91
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
|
|
104
|
-
else
|
|
105
|
-
logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
|
|
106
|
-
"on Linux platform. Browser will run in normal mode. Set `native` mode instead."
|
|
92
|
+
# Headless mode
|
|
93
|
+
if ENV["HEADLESS"] != "false"
|
|
94
|
+
if @config[:headless_mode] == :virtual_display
|
|
95
|
+
if Gem::Platform.local.os == "linux"
|
|
96
|
+
unless self.class.virtual_display
|
|
97
|
+
require 'headless'
|
|
98
|
+
self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
|
|
99
|
+
self.class.virtual_display.start
|
|
107
100
|
end
|
|
101
|
+
|
|
102
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
|
|
108
103
|
else
|
|
109
|
-
|
|
110
|
-
|
|
104
|
+
logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
|
|
105
|
+
"on Linux platform. Browser will run in normal mode. Set `native` mode instead."
|
|
111
106
|
end
|
|
107
|
+
else
|
|
108
|
+
driver_options.args << "--headless"
|
|
109
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled native headless_mode"
|
|
112
110
|
end
|
|
113
|
-
|
|
114
|
-
Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
|
|
115
111
|
end
|
|
116
112
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
@browser.spider = spider
|
|
120
|
-
logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
|
|
113
|
+
Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
|
|
114
|
+
end
|
|
121
115
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
116
|
+
# Create browser instance (Capybara session)
|
|
117
|
+
@browser = Capybara::Session.new(:selenium_firefox)
|
|
118
|
+
@browser.spider = spider
|
|
119
|
+
logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
|
|
125
120
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
|
|
130
|
-
end
|
|
121
|
+
if @config[:extensions].present?
|
|
122
|
+
logger.error "BrowserBuilder (selenium_firefox): `extensions` option not supported by Selenium, skipped"
|
|
123
|
+
end
|
|
131
124
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
125
|
+
# Window size
|
|
126
|
+
if size = @config[:window_size].presence
|
|
127
|
+
@browser.current_window.resize_to(*size)
|
|
128
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
|
|
129
|
+
end
|
|
137
130
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
end
|
|
131
|
+
# Cookies
|
|
132
|
+
if cookies = @config[:cookies].presence
|
|
133
|
+
@browser.config.cookies = cookies
|
|
134
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled custom cookies"
|
|
135
|
+
end
|
|
144
136
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
137
|
+
# Browser instance options
|
|
138
|
+
# skip_request_errors
|
|
139
|
+
if skip_errors = @config[:skip_request_errors].presence
|
|
140
|
+
@browser.config.skip_request_errors = skip_errors
|
|
141
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled skip_request_errors"
|
|
142
|
+
end
|
|
150
143
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
144
|
+
# retry_request_errors
|
|
145
|
+
if retry_errors = @config[:retry_request_errors].presence
|
|
146
|
+
@browser.config.retry_request_errors = retry_errors
|
|
147
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled retry_request_errors"
|
|
148
|
+
end
|
|
156
149
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
150
|
+
# restart_if
|
|
151
|
+
if requests_limit = @config.dig(:restart_if, :requests_limit).presence
|
|
152
|
+
@browser.config.restart_if[:requests_limit] = requests_limit
|
|
153
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.requests_limit >= #{requests_limit}"
|
|
154
|
+
end
|
|
161
155
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
end
|
|
156
|
+
if memory_limit = @config.dig(:restart_if, :memory_limit).presence
|
|
157
|
+
@browser.config.restart_if[:memory_limit] = memory_limit
|
|
158
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.memory_limit >= #{memory_limit}"
|
|
159
|
+
end
|
|
167
160
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
|
|
174
|
-
else
|
|
175
|
-
logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
|
|
176
|
-
end
|
|
177
|
-
end
|
|
161
|
+
# before_request clear_cookies
|
|
162
|
+
if @config.dig(:before_request, :clear_cookies)
|
|
163
|
+
@browser.config.before_request[:clear_cookies] = true
|
|
164
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_cookies"
|
|
165
|
+
end
|
|
178
166
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
167
|
+
# before_request clear_and_set_cookies
|
|
168
|
+
if @config.dig(:before_request, :clear_and_set_cookies)
|
|
169
|
+
if cookies = @config[:cookies].presence
|
|
170
|
+
@browser.config.cookies = cookies
|
|
171
|
+
@browser.config.before_request[:clear_and_set_cookies] = true
|
|
172
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
|
|
173
|
+
else
|
|
174
|
+
logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
|
|
182
175
|
end
|
|
176
|
+
end
|
|
183
177
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
178
|
+
# before_request change_user_agent
|
|
179
|
+
if @config.dig(:before_request, :change_user_agent)
|
|
180
|
+
logger.error "BrowserBuilder (selenium_firefox): before_request.change_user_agent option not supported by Selenium, skipped"
|
|
181
|
+
end
|
|
188
182
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
end
|
|
183
|
+
# before_request change_proxy
|
|
184
|
+
if @config.dig(:before_request, :change_proxy)
|
|
185
|
+
logger.error "BrowserBuilder (selenium_firefox): before_request.change_proxy option not supported by Selenium, skipped"
|
|
186
|
+
end
|
|
194
187
|
|
|
195
|
-
|
|
196
|
-
|
|
188
|
+
# before_request delay
|
|
189
|
+
if delay = @config.dig(:before_request, :delay).presence
|
|
190
|
+
@browser.config.before_request[:delay] = delay
|
|
191
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.delay"
|
|
197
192
|
end
|
|
193
|
+
|
|
194
|
+
# encoding
|
|
195
|
+
if encoding = @config[:encoding]
|
|
196
|
+
@browser.config.encoding = encoding
|
|
197
|
+
logger.debug "BrowserBuilder (selenium_firefox): enabled encoding: #{encoding}"
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# return Capybara session instance
|
|
201
|
+
@browser
|
|
198
202
|
end
|
|
199
203
|
end
|
|
200
204
|
end
|