kimurai 1.3.2 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,191 +4,195 @@ require_relative '../capybara_configuration'
4
4
  require_relative '../capybara_ext/selenium/driver'
5
5
  require_relative '../capybara_ext/session'
6
6
 
7
- module Kimurai
8
- class BrowserBuilder
9
- class SeleniumChromeBuilder
10
- class << self
11
- attr_accessor :virtual_display
12
- end
7
+ module Kimurai::BrowserBuilder
8
+ class SeleniumChromeBuilder
9
+ class << self
10
+ attr_accessor :virtual_display
11
+ end
13
12
 
14
- attr_reader :logger, :spider
13
+ attr_reader :logger, :spider
15
14
 
16
- def initialize(config, spider:)
17
- @config = config
18
- @spider = spider
19
- @logger = spider.logger
20
- end
15
+ def initialize(config, spider:)
16
+ @config = config
17
+ @spider = spider
18
+ @logger = spider.logger
19
+ end
21
20
 
22
- def build
23
- # Register driver
24
- Capybara.register_driver :selenium_chrome do |app|
25
- # Create driver options
26
- opts = { args: %w[--disable-gpu --no-sandbox --disable-translate] }
21
+ def build
22
+ # Register driver
23
+ Capybara.register_driver :selenium_chrome do |app|
24
+ # Create driver options
25
+ opts = { args: %w[--disable-gpu --no-sandbox --disable-translate] }
27
26
 
28
- # Provide custom chrome browser path:
29
- if chrome_path = Kimurai.configuration.selenium_chrome_path
30
- opts.merge!(binary: chrome_path)
31
- end
27
+ # Provide custom chrome browser path:
28
+ if chrome_path = Kimurai.configuration.selenium_chrome_path
29
+ opts.merge!(binary: chrome_path)
30
+ end
32
31
 
33
- # See all options here: https://seleniumhq.github.io/selenium/docs/api/rb/Selenium/WebDriver/Chrome/Options.html
34
- driver_options = Selenium::WebDriver::Chrome::Options.new(opts)
32
+ # See all options here: https://seleniumhq.github.io/selenium/docs/api/rb/Selenium/WebDriver/Chrome/Options.html
33
+ driver_options = Selenium::WebDriver::Chrome::Options.new(opts)
35
34
 
36
- # Window size
37
- if size = @config[:window_size].presence
38
- driver_options.args << "--window-size=#{size.join(',')}"
39
- logger.debug "BrowserBuilder (selenium_chrome): enabled window_size"
40
- end
35
+ # Window size
36
+ if size = @config[:window_size].presence
37
+ driver_options.args << "--window-size=#{size.join(',')}"
38
+ logger.debug "BrowserBuilder (selenium_chrome): enabled window_size"
39
+ end
41
40
 
42
- # Proxy
43
- if proxy = @config[:proxy].presence
44
- proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
45
- ip, port, type, user, password = proxy_string.split(":")
46
-
47
- if %w(http socks5).include?(type)
48
- if user.nil? && password.nil?
49
- driver_options.args << "--proxy-server=#{type}://#{ip}:#{port}"
50
- logger.debug "BrowserBuilder (selenium_chrome): enabled #{type} proxy, ip: #{ip}, port: #{port}"
51
- else
52
- logger.error "BrowserBuilder (selenium_chrome): proxy with authentication doesn't supported by selenium, skipped"
53
- end
54
- else
55
- logger.error "BrowserBuilder (selenium_chrome): wrong type of proxy: #{type}, skipped"
56
- end
57
- end
41
+ # Proxy
42
+ if proxy = @config[:proxy].presence
43
+ proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
44
+ ip, port, type, user, password = proxy_string.split(":")
58
45
 
59
- if proxy_bypass_list = @config[:proxy_bypass_list].presence
60
- if proxy
61
- driver_options.args << "--proxy-bypass-list=#{proxy_bypass_list.join(';')}"
62
- logger.debug "BrowserBuilder (selenium_chrome): enabled proxy_bypass_list"
46
+ if %w(http socks5).include?(type)
47
+ if user.nil? && password.nil?
48
+ driver_options.args << "--proxy-server=#{type}://#{ip}:#{port}"
49
+ logger.debug "BrowserBuilder (selenium_chrome): enabled #{type} proxy, ip: #{ip}, port: #{port}"
63
50
  else
64
- logger.error "BrowserBuilder (selenium_chrome): provide `proxy` to set proxy_bypass_list, skipped"
51
+ logger.error "BrowserBuilder (selenium_chrome): proxy with authentication doesn't supported by selenium, skipped"
65
52
  end
53
+ else
54
+ logger.error "BrowserBuilder (selenium_chrome): wrong type of proxy: #{type}, skipped"
66
55
  end
56
+ end
67
57
 
68
- # SSL
69
- if @config[:ignore_ssl_errors].present?
70
- driver_options.args << "--ignore-certificate-errors"
71
- driver_options.args << "--allow-insecure-localhost"
72
- logger.debug "BrowserBuilder (selenium_chrome): enabled ignore_ssl_errors"
58
+ if proxy_bypass_list = @config[:proxy_bypass_list].presence
59
+ if proxy
60
+ driver_options.args << "--proxy-bypass-list=#{proxy_bypass_list.join(';')}"
61
+ logger.debug "BrowserBuilder (selenium_chrome): enabled proxy_bypass_list"
62
+ else
63
+ logger.error "BrowserBuilder (selenium_chrome): provide `proxy` to set proxy_bypass_list, skipped"
73
64
  end
65
+ end
74
66
 
75
- # Disable images
76
- if @config[:disable_images].present?
77
- driver_options.prefs["profile.managed_default_content_settings.images"] = 2
78
- logger.debug "BrowserBuilder (selenium_chrome): enabled disable_images"
79
- end
67
+ # SSL
68
+ if @config[:ignore_ssl_errors].present?
69
+ driver_options.args << "--ignore-certificate-errors"
70
+ driver_options.args << "--allow-insecure-localhost"
71
+ logger.debug "BrowserBuilder (selenium_chrome): enabled ignore_ssl_errors"
72
+ end
80
73
 
81
- # Headers
82
- if @config[:headers].present?
83
- logger.warn "BrowserBuilder: (selenium_chrome): custom headers doesn't supported by selenium, skipped"
84
- end
74
+ # Disable images
75
+ if @config[:disable_images].present?
76
+ driver_options.prefs["profile.managed_default_content_settings.images"] = 2
77
+ logger.debug "BrowserBuilder (selenium_chrome): enabled disable_images"
78
+ end
85
79
 
86
- if user_agent = @config[:user_agent].presence
87
- user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
88
- driver_options.args << "--user-agent='#{user_agent_string}'"
89
- logger.debug "BrowserBuilder (selenium_chrome): enabled custom user_agent"
90
- end
80
+ # Headers
81
+ if @config[:headers].present?
82
+ logger.warn "BrowserBuilder: (selenium_chrome): custom headers doesn't supported by selenium, skipped"
83
+ end
91
84
 
92
- # Headless mode
93
- if ENV["HEADLESS"] != "false"
94
- if @config[:headless_mode] == :virtual_display
95
- if Gem::Platform.local.os == "linux"
96
- unless self.class.virtual_display
97
- require 'headless'
98
- self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
99
- self.class.virtual_display.start
100
- end
101
-
102
- logger.debug "BrowserBuilder (selenium_chrome): enabled virtual_display headless_mode"
103
- else
104
- logger.error "BrowserBuilder (selenium_chrome): virtual_display headless_mode works only " \
105
- "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
85
+ if user_agent = @config[:user_agent].presence
86
+ user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
87
+ driver_options.args << "--user-agent='#{user_agent_string}'"
88
+ logger.debug "BrowserBuilder (selenium_chrome): enabled custom user_agent"
89
+ end
90
+
91
+ # Headless mode
92
+ if ENV["HEADLESS"] != "false"
93
+ if @config[:headless_mode] == :virtual_display
94
+ if Gem::Platform.local.os == "linux"
95
+ unless self.class.virtual_display
96
+ require 'headless'
97
+ self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
98
+ self.class.virtual_display.start
106
99
  end
100
+
101
+ logger.debug "BrowserBuilder (selenium_chrome): enabled virtual_display headless_mode"
107
102
  else
108
- driver_options.args << "--headless"
109
- logger.debug "BrowserBuilder (selenium_chrome): enabled native headless_mode"
103
+ logger.error "BrowserBuilder (selenium_chrome): virtual_display headless_mode works only " \
104
+ "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
110
105
  end
106
+ else
107
+ driver_options.args << "--headless"
108
+ logger.debug "BrowserBuilder (selenium_chrome): enabled native headless_mode"
111
109
  end
112
-
113
- chromedriver_path = Kimurai.configuration.chromedriver_path || "/usr/local/bin/chromedriver"
114
- Capybara::Selenium::Driver.new(app, browser: :chrome, options: driver_options, driver_path: chromedriver_path)
115
110
  end
116
111
 
117
- # Create browser instance (Capybara session)
118
- @browser = Capybara::Session.new(:selenium_chrome)
119
- @browser.spider = spider
120
- logger.debug "BrowserBuilder (selenium_chrome): created browser instance"
112
+ chromedriver_path = Kimurai.configuration.chromedriver_path || "/usr/local/bin/chromedriver"
113
+ Capybara::Selenium::Driver.new(app, browser: :chrome, options: driver_options, driver_path: chromedriver_path)
114
+ end
121
115
 
122
- if @config[:extensions].present?
123
- logger.error "BrowserBuilder (selenium_chrome): `extensions` option not supported by Selenium, skipped"
124
- end
116
+ # Create browser instance (Capybara session)
117
+ @browser = Capybara::Session.new(:selenium_chrome)
118
+ @browser.spider = spider
119
+ logger.debug "BrowserBuilder (selenium_chrome): created browser instance"
125
120
 
126
- # Cookies
127
- if cookies = @config[:cookies].presence
128
- @browser.config.cookies = cookies
129
- logger.debug "BrowserBuilder (selenium_chrome): enabled custom cookies"
130
- end
121
+ if @config[:extensions].present?
122
+ logger.error "BrowserBuilder (selenium_chrome): `extensions` option not supported by Selenium, skipped"
123
+ end
131
124
 
132
- # Browser instance options
133
- # skip_request_errors
134
- if skip_errors = @config[:skip_request_errors].presence
135
- @browser.config.skip_request_errors = skip_errors
136
- logger.debug "BrowserBuilder (selenium_chrome): enabled skip_request_errors"
137
- end
125
+ # Cookies
126
+ if cookies = @config[:cookies].presence
127
+ @browser.config.cookies = cookies
128
+ logger.debug "BrowserBuilder (selenium_chrome): enabled custom cookies"
129
+ end
138
130
 
139
- # retry_request_errors
140
- if retry_errors = @config[:retry_request_errors].presence
141
- @browser.config.retry_request_errors = retry_errors
142
- logger.debug "BrowserBuilder (selenium_chrome): enabled retry_request_errors"
143
- end
131
+ # Browser instance options
132
+ # skip_request_errors
133
+ if skip_errors = @config[:skip_request_errors].presence
134
+ @browser.config.skip_request_errors = skip_errors
135
+ logger.debug "BrowserBuilder (selenium_chrome): enabled skip_request_errors"
136
+ end
144
137
 
145
- # restart_if
146
- if requests_limit = @config.dig(:restart_if, :requests_limit).presence
147
- @browser.config.restart_if[:requests_limit] = requests_limit
148
- logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.requests_limit >= #{requests_limit}"
149
- end
138
+ # retry_request_errors
139
+ if retry_errors = @config[:retry_request_errors].presence
140
+ @browser.config.retry_request_errors = retry_errors
141
+ logger.debug "BrowserBuilder (selenium_chrome): enabled retry_request_errors"
142
+ end
150
143
 
151
- if memory_limit = @config.dig(:restart_if, :memory_limit).presence
152
- @browser.config.restart_if[:memory_limit] = memory_limit
153
- logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.memory_limit >= #{memory_limit}"
154
- end
144
+ # restart_if
145
+ if requests_limit = @config.dig(:restart_if, :requests_limit).presence
146
+ @browser.config.restart_if[:requests_limit] = requests_limit
147
+ logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.requests_limit >= #{requests_limit}"
148
+ end
155
149
 
156
- # before_request clear_cookies
157
- if @config.dig(:before_request, :clear_cookies)
158
- @browser.config.before_request[:clear_cookies] = true
159
- logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_cookies"
160
- end
150
+ if memory_limit = @config.dig(:restart_if, :memory_limit).presence
151
+ @browser.config.restart_if[:memory_limit] = memory_limit
152
+ logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.memory_limit >= #{memory_limit}"
153
+ end
161
154
 
162
- # before_request clear_and_set_cookies
163
- if @config.dig(:before_request, :clear_and_set_cookies)
164
- if cookies = @config[:cookies].presence
165
- @browser.config.cookies = cookies
166
- @browser.config.before_request[:clear_and_set_cookies] = true
167
- logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_and_set_cookies"
168
- else
169
- logger.error "BrowserBuilder (selenium_chrome): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
170
- end
171
- end
155
+ # before_request clear_cookies
156
+ if @config.dig(:before_request, :clear_cookies)
157
+ @browser.config.before_request[:clear_cookies] = true
158
+ logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_cookies"
159
+ end
172
160
 
173
- # before_request change_user_agent
174
- if @config.dig(:before_request, :change_user_agent)
175
- logger.error "BrowserBuilder (selenium_chrome): before_request.change_user_agent option not supported by Selenium, skipped"
161
+ # before_request clear_and_set_cookies
162
+ if @config.dig(:before_request, :clear_and_set_cookies)
163
+ if cookies = @config[:cookies].presence
164
+ @browser.config.cookies = cookies
165
+ @browser.config.before_request[:clear_and_set_cookies] = true
166
+ logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_and_set_cookies"
167
+ else
168
+ logger.error "BrowserBuilder (selenium_chrome): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
176
169
  end
170
+ end
177
171
 
178
- # before_request change_proxy
179
- if @config.dig(:before_request, :change_proxy)
180
- logger.error "BrowserBuilder (selenium_chrome): before_request.change_proxy option not supported by Selenium, skipped"
181
- end
172
+ # before_request change_user_agent
173
+ if @config.dig(:before_request, :change_user_agent)
174
+ logger.error "BrowserBuilder (selenium_chrome): before_request.change_user_agent option not supported by Selenium, skipped"
175
+ end
182
176
 
183
- # before_request delay
184
- if delay = @config.dig(:before_request, :delay).presence
185
- @browser.config.before_request[:delay] = delay
186
- logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.delay"
187
- end
177
+ # before_request change_proxy
178
+ if @config.dig(:before_request, :change_proxy)
179
+ logger.error "BrowserBuilder (selenium_chrome): before_request.change_proxy option not supported by Selenium, skipped"
180
+ end
188
181
 
189
- # return Capybara session instance
190
- @browser
182
+ # before_request delay
183
+ if delay = @config.dig(:before_request, :delay).presence
184
+ @browser.config.before_request[:delay] = delay
185
+ logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.delay"
191
186
  end
187
+
188
+ # encoding
189
+ if encoding = @config[:encoding]
190
+ @browser.config.encoding = encoding
191
+ logger.debug "BrowserBuilder (selenium_chrome): enabled encoding: #{encoding}"
192
+ end
193
+
194
+ # return Capybara session instance
195
+ @browser
192
196
  end
193
197
  end
194
198
  end
@@ -4,197 +4,201 @@ require_relative '../capybara_configuration'
4
4
  require_relative '../capybara_ext/selenium/driver'
5
5
  require_relative '../capybara_ext/session'
6
6
 
7
- module Kimurai
8
- class BrowserBuilder
9
- class SeleniumFirefoxBuilder
10
- class << self
11
- attr_accessor :virtual_display
12
- end
7
+ module Kimurai::BrowserBuilder
8
+ class SeleniumFirefoxBuilder
9
+ class << self
10
+ attr_accessor :virtual_display
11
+ end
13
12
 
14
- attr_reader :logger, :spider
13
+ attr_reader :logger, :spider
15
14
 
16
- def initialize(config, spider:)
17
- @config = config
18
- @spider = spider
19
- @logger = spider.logger
20
- end
15
+ def initialize(config, spider:)
16
+ @config = config
17
+ @spider = spider
18
+ @logger = spider.logger
19
+ end
21
20
 
22
- def build
23
- # Register driver
24
- Capybara.register_driver :selenium_firefox do |app|
25
- # Create driver options
26
- driver_options = Selenium::WebDriver::Firefox::Options.new
27
- driver_options.profile = Selenium::WebDriver::Firefox::Profile.new
28
- driver_options.profile["browser.link.open_newwindow"] = 3 # open windows in tabs
29
- driver_options.profile["media.peerconnection.enabled"] = false # disable web rtc
30
-
31
- # Proxy
32
- if proxy = @config[:proxy].presence
33
- proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
34
- ip, port, type, user, password = proxy_string.split(":")
35
-
36
- if user.nil? && password.nil?
37
- driver_options.profile["network.proxy.type"] = 1
38
- if type == "http"
39
- driver_options.profile["network.proxy.http"] = ip
40
- driver_options.profile["network.proxy.http_port"] = port.to_i
41
- driver_options.profile["network.proxy.ssl"] = ip
42
- driver_options.profile["network.proxy.ssl_port"] = port.to_i
43
-
44
- logger.debug "BrowserBuilder (selenium_firefox): enabled http proxy, ip: #{ip}, port: #{port}"
45
- elsif type == "socks5"
46
- driver_options.profile["network.proxy.socks"] = ip
47
- driver_options.profile["network.proxy.socks_port"] = port.to_i
48
- driver_options.profile["network.proxy.socks_version"] = 5
49
- driver_options.profile["network.proxy.socks_remote_dns"] = true
50
-
51
- logger.debug "BrowserBuilder (selenium_firefox): enabled socks5 proxy, ip: #{ip}, port: #{port}"
52
- else
53
- logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
54
- end
21
+ def build
22
+ # Register driver
23
+ Capybara.register_driver :selenium_firefox do |app|
24
+ # Create driver options
25
+ driver_options = Selenium::WebDriver::Firefox::Options.new
26
+ driver_options.profile = Selenium::WebDriver::Firefox::Profile.new
27
+ driver_options.profile["browser.link.open_newwindow"] = 3 # open windows in tabs
28
+ driver_options.profile["media.peerconnection.enabled"] = false # disable web rtc
29
+
30
+ # Proxy
31
+ if proxy = @config[:proxy].presence
32
+ proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
33
+ ip, port, type, user, password = proxy_string.split(":")
34
+
35
+ if user.nil? && password.nil?
36
+ driver_options.profile["network.proxy.type"] = 1
37
+ if type == "http"
38
+ driver_options.profile["network.proxy.http"] = ip
39
+ driver_options.profile["network.proxy.http_port"] = port.to_i
40
+ driver_options.profile["network.proxy.ssl"] = ip
41
+ driver_options.profile["network.proxy.ssl_port"] = port.to_i
42
+
43
+ logger.debug "BrowserBuilder (selenium_firefox): enabled http proxy, ip: #{ip}, port: #{port}"
44
+ elsif type == "socks5"
45
+ driver_options.profile["network.proxy.socks"] = ip
46
+ driver_options.profile["network.proxy.socks_port"] = port.to_i
47
+ driver_options.profile["network.proxy.socks_version"] = 5
48
+ driver_options.profile["network.proxy.socks_remote_dns"] = true
49
+
50
+ logger.debug "BrowserBuilder (selenium_firefox): enabled socks5 proxy, ip: #{ip}, port: #{port}"
55
51
  else
56
- logger.error "BrowserBuilder (selenium_firefox): proxy with authentication doesn't supported by selenium, skipped"
52
+ logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
57
53
  end
54
+ else
55
+ logger.error "BrowserBuilder (selenium_firefox): proxy with authentication doesn't supported by selenium, skipped"
58
56
  end
57
+ end
59
58
 
60
- if proxy_bypass_list = @config[:proxy_bypass_list].presence
61
- if proxy
62
- driver_options.profile["network.proxy.no_proxies_on"] = proxy_bypass_list.join(", ")
63
- logger.debug "BrowserBuilder (selenium_firefox): enabled proxy_bypass_list"
64
- else
65
- logger.error "BrowserBuilder (selenium_firefox): provide `proxy` to set proxy_bypass_list, skipped"
66
- end
59
+ if proxy_bypass_list = @config[:proxy_bypass_list].presence
60
+ if proxy
61
+ driver_options.profile["network.proxy.no_proxies_on"] = proxy_bypass_list.join(", ")
62
+ logger.debug "BrowserBuilder (selenium_firefox): enabled proxy_bypass_list"
63
+ else
64
+ logger.error "BrowserBuilder (selenium_firefox): provide `proxy` to set proxy_bypass_list, skipped"
67
65
  end
66
+ end
68
67
 
69
- # SSL
70
- if @config[:ignore_ssl_errors].present?
71
- driver_options.profile.secure_ssl = false
72
- driver_options.profile.assume_untrusted_certificate_issuer = true
73
- logger.debug "BrowserBuilder (selenium_firefox): enabled ignore_ssl_errors"
74
- end
68
+ # SSL
69
+ if @config[:ignore_ssl_errors].present?
70
+ driver_options.profile.secure_ssl = false
71
+ driver_options.profile.assume_untrusted_certificate_issuer = true
72
+ logger.debug "BrowserBuilder (selenium_firefox): enabled ignore_ssl_errors"
73
+ end
75
74
 
76
- # Disable images
77
- if @config[:disable_images].present?
78
- driver_options.profile["permissions.default.image"] = 2
79
- logger.debug "BrowserBuilder (selenium_firefox): enabled disable_images"
80
- end
75
+ # Disable images
76
+ if @config[:disable_images].present?
77
+ driver_options.profile["permissions.default.image"] = 2
78
+ logger.debug "BrowserBuilder (selenium_firefox): enabled disable_images"
79
+ end
81
80
 
82
- # Headers
83
- if @config[:headers].present?
84
- logger.warn "BrowserBuilder: (selenium_firefox): custom headers doesn't supported by selenium, skipped"
85
- end
81
+ # Headers
82
+ if @config[:headers].present?
83
+ logger.warn "BrowserBuilder: (selenium_firefox): custom headers doesn't supported by selenium, skipped"
84
+ end
86
85
 
87
- if user_agent = @config[:user_agent].presence
88
- user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
89
- driver_options.profile["general.useragent.override"] = user_agent_string
90
- logger.debug "BrowserBuilder (selenium_firefox): enabled custom user_agent"
91
- end
86
+ if user_agent = @config[:user_agent].presence
87
+ user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
88
+ driver_options.profile["general.useragent.override"] = user_agent_string
89
+ logger.debug "BrowserBuilder (selenium_firefox): enabled custom user_agent"
90
+ end
92
91
 
93
- # Headless mode
94
- if ENV["HEADLESS"] != "false"
95
- if @config[:headless_mode] == :virtual_display
96
- if Gem::Platform.local.os == "linux"
97
- unless self.class.virtual_display
98
- require 'headless'
99
- self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
100
- self.class.virtual_display.start
101
- end
102
-
103
- logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
104
- else
105
- logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
106
- "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
92
+ # Headless mode
93
+ if ENV["HEADLESS"] != "false"
94
+ if @config[:headless_mode] == :virtual_display
95
+ if Gem::Platform.local.os == "linux"
96
+ unless self.class.virtual_display
97
+ require 'headless'
98
+ self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
99
+ self.class.virtual_display.start
107
100
  end
101
+
102
+ logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
108
103
  else
109
- driver_options.args << "--headless"
110
- logger.debug "BrowserBuilder (selenium_firefox): enabled native headless_mode"
104
+ logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
105
+ "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
111
106
  end
107
+ else
108
+ driver_options.args << "--headless"
109
+ logger.debug "BrowserBuilder (selenium_firefox): enabled native headless_mode"
112
110
  end
113
-
114
- Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
115
111
  end
116
112
 
117
- # Create browser instance (Capybara session)
118
- @browser = Capybara::Session.new(:selenium_firefox)
119
- @browser.spider = spider
120
- logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
113
+ Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
114
+ end
121
115
 
122
- if @config[:extensions].present?
123
- logger.error "BrowserBuilder (selenium_firefox): `extensions` option not supported by Selenium, skipped"
124
- end
116
+ # Create browser instance (Capybara session)
117
+ @browser = Capybara::Session.new(:selenium_firefox)
118
+ @browser.spider = spider
119
+ logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
125
120
 
126
- # Window size
127
- if size = @config[:window_size].presence
128
- @browser.current_window.resize_to(*size)
129
- logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
130
- end
121
+ if @config[:extensions].present?
122
+ logger.error "BrowserBuilder (selenium_firefox): `extensions` option not supported by Selenium, skipped"
123
+ end
131
124
 
132
- # Cookies
133
- if cookies = @config[:cookies].presence
134
- @browser.config.cookies = cookies
135
- logger.debug "BrowserBuilder (selenium_firefox): enabled custom cookies"
136
- end
125
+ # Window size
126
+ if size = @config[:window_size].presence
127
+ @browser.current_window.resize_to(*size)
128
+ logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
129
+ end
137
130
 
138
- # Browser instance options
139
- # skip_request_errors
140
- if skip_errors = @config[:skip_request_errors].presence
141
- @browser.config.skip_request_errors = skip_errors
142
- logger.debug "BrowserBuilder (selenium_firefox): enabled skip_request_errors"
143
- end
131
+ # Cookies
132
+ if cookies = @config[:cookies].presence
133
+ @browser.config.cookies = cookies
134
+ logger.debug "BrowserBuilder (selenium_firefox): enabled custom cookies"
135
+ end
144
136
 
145
- # retry_request_errors
146
- if retry_errors = @config[:retry_request_errors].presence
147
- @browser.config.retry_request_errors = retry_errors
148
- logger.debug "BrowserBuilder (selenium_firefox): enabled retry_request_errors"
149
- end
137
+ # Browser instance options
138
+ # skip_request_errors
139
+ if skip_errors = @config[:skip_request_errors].presence
140
+ @browser.config.skip_request_errors = skip_errors
141
+ logger.debug "BrowserBuilder (selenium_firefox): enabled skip_request_errors"
142
+ end
150
143
 
151
- # restart_if
152
- if requests_limit = @config.dig(:restart_if, :requests_limit).presence
153
- @browser.config.restart_if[:requests_limit] = requests_limit
154
- logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.requests_limit >= #{requests_limit}"
155
- end
144
+ # retry_request_errors
145
+ if retry_errors = @config[:retry_request_errors].presence
146
+ @browser.config.retry_request_errors = retry_errors
147
+ logger.debug "BrowserBuilder (selenium_firefox): enabled retry_request_errors"
148
+ end
156
149
 
157
- if memory_limit = @config.dig(:restart_if, :memory_limit).presence
158
- @browser.config.restart_if[:memory_limit] = memory_limit
159
- logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.memory_limit >= #{memory_limit}"
160
- end
150
+ # restart_if
151
+ if requests_limit = @config.dig(:restart_if, :requests_limit).presence
152
+ @browser.config.restart_if[:requests_limit] = requests_limit
153
+ logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.requests_limit >= #{requests_limit}"
154
+ end
161
155
 
162
- # before_request clear_cookies
163
- if @config.dig(:before_request, :clear_cookies)
164
- @browser.config.before_request[:clear_cookies] = true
165
- logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_cookies"
166
- end
156
+ if memory_limit = @config.dig(:restart_if, :memory_limit).presence
157
+ @browser.config.restart_if[:memory_limit] = memory_limit
158
+ logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.memory_limit >= #{memory_limit}"
159
+ end
167
160
 
168
- # before_request clear_and_set_cookies
169
- if @config.dig(:before_request, :clear_and_set_cookies)
170
- if cookies = @config[:cookies].presence
171
- @browser.config.cookies = cookies
172
- @browser.config.before_request[:clear_and_set_cookies] = true
173
- logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
174
- else
175
- logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
176
- end
177
- end
161
+ # before_request clear_cookies
162
+ if @config.dig(:before_request, :clear_cookies)
163
+ @browser.config.before_request[:clear_cookies] = true
164
+ logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_cookies"
165
+ end
178
166
 
179
- # before_request change_user_agent
180
- if @config.dig(:before_request, :change_user_agent)
181
- logger.error "BrowserBuilder (selenium_firefox): before_request.change_user_agent option not supported by Selenium, skipped"
167
+ # before_request clear_and_set_cookies
168
+ if @config.dig(:before_request, :clear_and_set_cookies)
169
+ if cookies = @config[:cookies].presence
170
+ @browser.config.cookies = cookies
171
+ @browser.config.before_request[:clear_and_set_cookies] = true
172
+ logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
173
+ else
174
+ logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
182
175
  end
176
+ end
183
177
 
184
- # before_request change_proxy
185
- if @config.dig(:before_request, :change_proxy)
186
- logger.error "BrowserBuilder (selenium_firefox): before_request.change_proxy option not supported by Selenium, skipped"
187
- end
178
+ # before_request change_user_agent
179
+ if @config.dig(:before_request, :change_user_agent)
180
+ logger.error "BrowserBuilder (selenium_firefox): before_request.change_user_agent option not supported by Selenium, skipped"
181
+ end
188
182
 
189
- # before_request delay
190
- if delay = @config.dig(:before_request, :delay).presence
191
- @browser.config.before_request[:delay] = delay
192
- logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.delay"
193
- end
183
+ # before_request change_proxy
184
+ if @config.dig(:before_request, :change_proxy)
185
+ logger.error "BrowserBuilder (selenium_firefox): before_request.change_proxy option not supported by Selenium, skipped"
186
+ end
194
187
 
195
- # return Capybara session instance
196
- @browser
188
+ # before_request delay
189
+ if delay = @config.dig(:before_request, :delay).presence
190
+ @browser.config.before_request[:delay] = delay
191
+ logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.delay"
197
192
  end
193
+
194
+ # encoding
195
+ if encoding = @config[:encoding]
196
+ @browser.config.encoding = encoding
197
+ logger.debug "BrowserBuilder (selenium_firefox): enabled encoding: #{encoding}"
198
+ end
199
+
200
+ # return Capybara session instance
201
+ @browser
198
202
  end
199
203
  end
200
204
  end