ferrum 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 3062988d83fbb92a5b52b9037fdc73a66f79cefe7e8c46e776642c805b9595ba
4
+ data.tar.gz: 150abbfd5781c22988db1d15432a9ee45a0180c4ca81691ecdef1ca8febcfcd2
5
+ SHA512:
6
+ metadata.gz: 6b2bb8783725639bc80103673725ad9d9573d58ab2ab276ab8a9347867a6782eca177195a9418f4dee64aa1b91aa1c196b423b2e097106aa78513df7659a4d93
7
+ data.tar.gz: 282beb51703405e569501a2e0c17f1a85aae7b03ee9342df90bca056232e9c60b9f31cbfdd27b8b947999731b759f0f7e39f1bc505c0ddad100af3f2dd52e8e5
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 Dmitry Vorotilin
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,28 @@
1
+ # Ferrum - fearless Ruby Chrome/Chromium driver.
2
+
3
+ As simple as Puppeteer, though even simpler. It is Ruby clean and high-level API
4
+ to Chrome/Chromium through the DevTools Protocol. Runs headless by default,
5
+ but you can configure it to run in a non-headless mode.
6
+
7
+ Navigate to `example.com` and save a screenshot:
8
+
9
+ ```ruby
10
+ browser = Ferrum::Browser.new
11
+ browser.goto("https://example.com")
12
+ browser.screenshot(path: "example.png")
13
+ browser.quit
14
+ ```
15
+
16
+ Interact with a page:
17
+
18
+ ```ruby
19
+ browser = Ferrum::Browser.new
20
+ browser.goto("https://google.com")
21
+ input = browser.at_css("input[title='Search']")
22
+ input.send_keys("Ruby headless driver for Capybara")
23
+ input.send_keys(:Enter)
24
+ browser.at_css("a > h3").text # => "machinio/cuprite: Headless Chrome driver for Capybara - GitHub"
25
+ browser.quit
26
+ ```
27
+
28
+ The README will be updated soon. Meanwhile take a look at specs.
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ Thread.abort_on_exception = true
4
+ Thread.report_on_exception = true if Thread.respond_to?(:report_on_exception=)
5
+
6
+ module Ferrum
7
+ require "ferrum/browser"
8
+ require "ferrum/node"
9
+ require "ferrum/errors"
10
+ require "ferrum/cookie"
11
+
12
+ class << self
13
+ def windows?
14
+ RbConfig::CONFIG["host_os"] =~ /mingw|mswin|cygwin/
15
+ end
16
+
17
+ def mac?
18
+ RbConfig::CONFIG["host_os"] =~ /darwin/
19
+ end
20
+
21
+ def mri?
22
+ defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby"
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,145 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "base64"
4
+ require "forwardable"
5
+ require "ferrum/page"
6
+ require "ferrum/targets"
7
+ require "ferrum/browser/api"
8
+ require "ferrum/browser/process"
9
+ require "ferrum/browser/client"
10
+
11
+ module Ferrum
12
+ class Browser
13
+ TIMEOUT = 5
14
+ WINDOW_SIZE = [1024, 768].freeze
15
+ BASE_URL_SCHEMA = %w[http https].freeze
16
+
17
+ include API
18
+ extend Forwardable
19
+
20
+ attr_reader :headers, :window_size
21
+
22
+ delegate on: :@client
23
+ delegate %i(window_handle window_handles switch_to_window open_new_window
24
+ close_window within_window page) => :targets
25
+ delegate %i(goto status body at_css at_xpath css xpath text property attributes attribute select_file
26
+ value visible? disabled? network_traffic clear_network_traffic
27
+ path response_headers refresh click right_click double_click
28
+ hover set click_coordinates select trigger scroll_to send_keys
29
+ evaluate evaluate_on evaluate_async execute frame_url
30
+ frame_title switch_to_frame current_url title go_back
31
+ go_forward find_modal accept_confirm dismiss_confirm
32
+ accept_prompt dismiss_prompt reset_modals authorize
33
+ proxy_authorize) => :page
34
+
35
+ attr_reader :process, :logger, :js_errors, :slowmo, :base_url,
36
+ :url_blacklist, :url_whitelist, :options
37
+ attr_writer :timeout
38
+
39
+ def initialize(options = nil)
40
+ options ||= {}
41
+
42
+ @client = nil
43
+ @window_size = options.fetch(:window_size, WINDOW_SIZE)
44
+ @original_window_size = @window_size
45
+
46
+ @options = Hash(options.merge(window_size: @window_size))
47
+ @logger, @timeout = @options.values_at(:logger, :timeout)
48
+ @js_errors = @options.fetch(:js_errors, false)
49
+ @slowmo = @options[:slowmo].to_i
50
+
51
+ if @options.key?(:base_url)
52
+ self.base_url = @options[:base_url]
53
+ end
54
+
55
+ self.url_blacklist = @options[:url_blacklist]
56
+ self.url_whitelist = @options[:url_whitelist]
57
+
58
+ if ENV["FERRUM_DEBUG"] && !@logger
59
+ STDOUT.sync = true
60
+ @logger = STDOUT
61
+ @options[:logger] = @logger
62
+ end
63
+
64
+ @options.freeze
65
+
66
+ start
67
+ end
68
+
69
+ def base_url=(value)
70
+ parsed = Addressable::URI.parse(value)
71
+ unless BASE_URL_SCHEMA.include?(parsed.normalized_scheme)
72
+ raise "Set `base_url` should be absolute and include schema: #{BASE_URL_SCHEMA}"
73
+ end
74
+
75
+ @base_url = parsed
76
+ end
77
+
78
+ def extensions
79
+ @extensions ||= Array(@options[:extensions]).map { |p| File.read(p) }
80
+ end
81
+
82
+ def timeout
83
+ @timeout || TIMEOUT
84
+ end
85
+
86
+ def command(*args)
87
+ @client.command(*args)
88
+ rescue DeadBrowser
89
+ restart
90
+ raise
91
+ end
92
+
93
+ def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
94
+ options = Hash.new
95
+ options[:userAgent] = user_agent if user_agent
96
+ options[:acceptLanguage] = accept_language if accept_language
97
+ options[:platform] if platform
98
+
99
+ page.command("Network.setUserAgentOverride", **options) if !options.empty?
100
+ end
101
+
102
+ def clear_memory_cache
103
+ page.command("Network.clearBrowserCache")
104
+ end
105
+
106
+ def reset
107
+ @headers = {}
108
+ @zoom_factor = nil
109
+ @window_size = @original_window_size
110
+ targets.reset
111
+ end
112
+
113
+ def restart
114
+ quit
115
+ start
116
+ end
117
+
118
+ def quit
119
+ @client.close
120
+ @process.stop
121
+ @client = @process = @targets = nil
122
+ end
123
+
124
+ def targets
125
+ @targets ||= Targets.new(self)
126
+ end
127
+
128
+ def resize(**options)
129
+ @window_size = [options[:width], options[:height]]
130
+ page.resize(**options)
131
+ end
132
+
133
+ def crash
134
+ command("Browser.crash")
135
+ end
136
+
137
+ private
138
+
139
+ def start
140
+ @headers = {}
141
+ @process = Process.start(@options)
142
+ @client = Client.new(self, @process.ws_url, 0, false)
143
+ end
144
+ end
145
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ferrum/browser/api/cookie"
4
+ require "ferrum/browser/api/header"
5
+ require "ferrum/browser/api/screenshot"
6
+ require "ferrum/browser/api/intercept"
7
+
8
+ module Ferrum
9
+ class Browser
10
+ module API
11
+ include Cookie, Header, Screenshot, Intercept
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Browser
5
+ module API
6
+ module Cookie
7
+ def cookies
8
+ cookies = page.command("Network.getAllCookies")["cookies"]
9
+ cookies.map { |c| [c["name"], ::Ferrum::Cookie.new(c)] }.to_h
10
+ end
11
+
12
+ def set_cookie(name: nil, value: nil, cookie: nil, **options)
13
+ cookie = options.dup
14
+ cookie[:name] ||= name
15
+ cookie[:value] ||= value
16
+ cookie[:domain] ||= default_domain
17
+
18
+ expires = cookie.delete(:expires).to_i
19
+ cookie[:expires] = expires if expires > 0
20
+
21
+ page.command("Network.setCookie", **cookie)
22
+ end
23
+
24
+ # Supports :url, :domain and :path options
25
+ def remove_cookie(name:, **options)
26
+ raise "Specify :domain or :url option" if !options[:domain] && !options[:url] && !default_domain
27
+
28
+ options = options.merge(name: name)
29
+ options[:domain] ||= default_domain
30
+
31
+ page.command("Network.deleteCookies", **options)
32
+ end
33
+
34
+ def clear_cookies
35
+ page.command("Network.clearBrowserCookies")
36
+ end
37
+
38
+ private
39
+
40
+ def default_domain
41
+ URI.parse(base_url).host if base_url
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Browser
5
+ module API
6
+ module Header
7
+ def headers=(headers)
8
+ @headers = {}
9
+ add_headers(headers)
10
+ end
11
+
12
+ def add_headers(headers, permanent: true)
13
+ if headers["Referer"]
14
+ page.referrer = headers["Referer"]
15
+ headers.delete("Referer") unless permanent
16
+ end
17
+
18
+ @headers.merge!(headers)
19
+ user_agent = @headers["User-Agent"]
20
+ accept_language = @headers["Accept-Language"]
21
+
22
+ set_overrides(user_agent: user_agent, accept_language: accept_language)
23
+ page.command("Network.setExtraHTTPHeaders", headers: @headers)
24
+ end
25
+
26
+ def add_header(header, permanent: true)
27
+ add_headers(header, permanent: permanent)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Browser
5
+ module API
6
+ module Intercept
7
+ def url_whitelist=(wildcards)
8
+ @url_whitelist = prepare_wildcards(wildcards)
9
+ page.intercept_request("*") if @client && !@url_whitelist.empty?
10
+ end
11
+
12
+ def url_blacklist=(wildcards)
13
+ @url_blacklist = prepare_wildcards(wildcards)
14
+ page.intercept_request("*") if @client && !@url_blacklist.empty?
15
+ end
16
+
17
+ private
18
+
19
+ def prepare_wildcards(wc)
20
+ Array(wc).map do |wildcard|
21
+ if wildcard.is_a?(Regexp)
22
+ wildcard
23
+ else
24
+ wildcard = wildcard.gsub("*", ".*")
25
+ Regexp.new(wildcard, Regexp::IGNORECASE)
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Browser
5
+ module API
6
+ module Screenshot
7
+ def screenshot(**opts)
8
+ encoding, path, options = screenshot_options(**opts)
9
+
10
+ data = if options[:format].to_s == "pdf"
11
+ options = {}
12
+ options[:paperWidth] = @paper_size[:width].to_f if @paper_size
13
+ options[:paperHeight] = @paper_size[:height].to_f if @paper_size
14
+ options[:scale] = @zoom_factor if @zoom_factor
15
+ page.command("Page.printToPDF", **options)
16
+ else
17
+ page.command("Page.captureScreenshot", **options)
18
+ end.fetch("data")
19
+
20
+ return data if encoding == :base64
21
+
22
+ bin = Base64.decode64(data)
23
+ File.open(path.to_s, "wb") { |f| f.write(bin) }
24
+ end
25
+
26
+ def zoom_factor=(value)
27
+ @zoom_factor = value.to_f
28
+ end
29
+
30
+ def paper_size=(value)
31
+ @paper_size = value
32
+ end
33
+
34
+ private
35
+
36
+ def screenshot_options(encoding: :base64, format: nil, path: nil, **opts)
37
+ options = {}
38
+
39
+ encoding = :binary if path
40
+
41
+ if encoding == :binary && !path
42
+ raise "Not supported option `:path` #{path}. Should be path to file"
43
+ end
44
+
45
+ format ||= path ? File.extname(path).delete(".") : "png"
46
+ format = "jpeg" if format == "jpg"
47
+ raise "Not supported options `:format` #{format}. jpeg | png | pdf" if format !~ /jpeg|png|pdf/i
48
+ options.merge!(format: format)
49
+
50
+ options.merge!(quality: opts[:quality] ? opts[:quality] : 75) if format == "jpeg"
51
+
52
+ if !!opts[:full] && opts[:selector]
53
+ warn "Ignoring :selector in #screenshot since full: true was given at #{caller(1..1).first}"
54
+ end
55
+
56
+ if !!opts[:full]
57
+ width, height = page.evaluate("[document.documentElement.offsetWidth, document.documentElement.offsetHeight]")
58
+ options.merge!(clip: { x: 0, y: 0, width: width, height: height, scale: @zoom_factor || 1.0 }) if width > 0 && height > 0
59
+ elsif opts[:selector]
60
+ rect = page.evaluate("document.querySelector('#{opts[:selector]}').getBoundingClientRect()")
61
+ options.merge!(clip: { x: rect["x"], y: rect["y"], width: rect["width"], height: rect["height"], scale: @zoom_factor || 1.0 })
62
+ end
63
+
64
+ if @zoom_factor
65
+ if !options[:clip]
66
+ width, height = page.evaluate("[document.documentElement.clientWidth, document.documentElement.clientHeight]")
67
+ options[:clip] = { x: 0, y: 0, width: width, height: height }
68
+ end
69
+
70
+ options[:clip].merge!(scale: @zoom_factor)
71
+ end
72
+
73
+ [encoding, path, options]
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "concurrent-ruby"
4
+ require "ferrum/browser/subscriber"
5
+ require "ferrum/browser/web_socket"
6
+
7
+ module Ferrum
8
+ class Browser
9
+ class Client
10
+ def initialize(browser, ws_url, start_id = 0, allow_slowmo = true)
11
+ @command_id = start_id
12
+ @pendings = Concurrent::Hash.new
13
+ @browser = browser
14
+ @slowmo = @browser.slowmo if allow_slowmo && @browser.slowmo > 0
15
+ @ws = WebSocket.new(ws_url, @browser.logger)
16
+ @subscriber = Subscriber.new
17
+
18
+ @thread = Thread.new do
19
+ while message = @ws.messages.pop
20
+ if message.key?("method")
21
+ @subscriber.async.call(message)
22
+ else
23
+ @pendings[message["id"]]&.set(message)
24
+ end
25
+ end
26
+ end
27
+ end
28
+
29
+ def command(method, params = {})
30
+ pending = Concurrent::IVar.new
31
+ message = build_message(method, params)
32
+ @pendings[message[:id]] = pending
33
+ sleep(@slowmo) if @slowmo
34
+ @ws.send_message(message)
35
+ data = pending.value!(@browser.timeout)
36
+ @pendings.delete(message[:id])
37
+
38
+ raise DeadBrowser if data.nil? && @ws.messages.closed?
39
+ raise TimeoutError unless data
40
+ error, response = data.values_at("error", "result")
41
+ raise BrowserError.new(error) if error
42
+ response
43
+ end
44
+
45
+ def on(event, &block)
46
+ @subscriber.on(event, &block)
47
+ end
48
+
49
+ def close
50
+ @ws.close
51
+ # Give a thread some time to handle a tail of messages
52
+ @pendings.clear
53
+ Timeout.timeout(1) { @thread.join }
54
+ rescue Timeout::Error
55
+ @thread.kill
56
+ end
57
+
58
+ private
59
+
60
+ def build_message(method, params)
61
+ { method: method, params: params }.merge(id: next_command_id)
62
+ end
63
+
64
+ def next_command_id
65
+ @command_id += 1
66
+ end
67
+ end
68
+ end
69
+ end