ferrum 0.4 → 0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f524fb695add05ab04355a4f694710f0cda6ce317048f234cd6d597a6db5b54
-  data.tar.gz: 4f755ea3ce87efa0a92b2689188ec8ce78979b716e8053e9687fe4f9e97baf2e
+  metadata.gz: 703207ec942dcaa7a09a6a2436e9a358da43a4f44e195150814ff1d00e05e3f8
+  data.tar.gz: 27e7a22921a96e1fbf4d36215170fb884e4ec72e904686cb12eae014f6dfd0a6
 SHA512:
-  metadata.gz: f52f33ea6d6292ab9f829f9e4870ba7be5fe0f9c045f4b4fe1e540b5363440027ae8890987e92e95612b91e1d82d28a19ff3d204cabeac0bebde96943d399baa
-  data.tar.gz: 3156ae9cc12403dd2b497119371b895d7f37865318ffa2d424676df91346403d85386615abe2db020dffe77b055de72b5e44ccf69137d9bd1fa01a9ae75ffa0f
+  metadata.gz: b241f4500a4d330b2da5209e789a20c0df95f37963a6031846f51e8950378885749a2c3217116ef37dc09a5efcc421eaf4b1301536a38b9eb0747f2e1037f904
+  data.tar.gz: 60abc95e3633224aa08923da5b78026a687d21f380b3a818b99dcc1f32950fce83a7de574276f21973360ea257aaa8560490202e735953cbd6b714466eeefee8

data/README.md

@@ -23,10 +23,12 @@ There's no official Chrome or Chromium package for Linux don't install it this
 way because it either will be outdated or unofficial, both are bad. Download it
 from official [source](https://www.chromium.org/getting-involved/download-chromium).
 Chrome binary should be in the `PATH` or `BROWSER_PATH` or you can pass it as an
-option.
+option to browser instance `:browser_path`.
+
+Add this to your Gemfile:
 
 ``` ruby
-gem install "ferrum"
+gem "ferrum"
 ```
 
 Navigate to a website and save a screenshot:
@@ -641,3 +643,61 @@ browser.on(:dialog) do |dialog|
 end
 browser.goto("https://google.com")
 ```
+
+
+## Thread safety ##
+
+Ferrum is fully thread-safe. You can create one browser or a few as you wish and
+start playing around using threads. Example below shows how to create a few pages
+which share the same context. Context is similar to an incognito profile but you
+can have more than one, think of it like it's independent browser session:
+
+```ruby
+browser = Ferrum::Browser.new
+context = browser.contexts.create
+
+t1 = Thread.new(context) do |c|
+  page = c.create_page
+  page.goto("https://www.google.com/search?q=Ruby+headless+driver+for+Capybara")
+  page.screenshot(path: "t1.png")
+end
+
+t2 = Thread.new(context) do |c|
+  page = c.create_page
+  page.goto("https://www.google.com/search?q=Ruby+static+typing")
+  page.screenshot(path: "t2.png")
+end
+
+t1.join
+t2.join
+
+context.dispose
+browser.quit
+```
+
+or you can create two independent contexts:
+
+```ruby
+browser = Ferrum::Browser.new
+
+t1 = Thread.new(browser) do |b|
+  context = b.contexts.create
+  page = context.create_page
+  page.goto("https://www.google.com/search?q=Ruby+headless+driver+for+Capybara")
+  page.screenshot(path: "t1.png")
+  context.dispose
+end
+
+t2 = Thread.new(browser) do |b|
+  context = b.contexts.create
+  page = context.create_page
+  page.goto("https://www.google.com/search?q=Ruby+static+typing")
+  page.screenshot(path: "t2.png")
+  context.dispose
+end
+
+t1.join
+t2.join
+
+browser.quit
+```

data/lib/ferrum.rb

@@ -5,15 +5,10 @@ require "ferrum/node"
 
 module Ferrum
   class Error               < StandardError; end
-  class NoSuchWindowError   < Error; end
+  class NoSuchPageError     < Error; end
+  class NoSuchTargetError   < Error; end
   class NotImplementedError < Error; end
 
-  class EmptyTargetsError < Error
-    def initialize
-      super("There aren't targets available")
-    end
-  end
-
   class StatusError < Error
     def initialize(url)
       super("Request to #{url} failed to reach server, check DNS and/or server status")

data/lib/ferrum/browser.rb

@@ -3,7 +3,7 @@
 require "base64"
 require "forwardable"
 require "ferrum/page"
-require "ferrum/targets"
+require "ferrum/contexts"
 require "ferrum/browser/process"
 require "ferrum/browser/client"
 
@@ -14,19 +14,19 @@ module Ferrum
     BASE_URL_SCHEMA = %w[http https].freeze
 
     extend Forwardable
-    delegate %i[window_handle window_handles switch_to_window
-                open_new_window close_window within_window page] => :targets
+    delegate %i[default_context] => :contexts
+    delegate %i[targets create_target create_page page pages windows] => :default_context
     delegate %i[goto back forward refresh
                 at_css at_xpath css xpath current_url title body
                 headers cookies network
                 mouse keyboard
-                screenshot pdf
+                screenshot pdf viewport_size
                 evaluate evaluate_on evaluate_async execute
                 frame_url frame_title within_frame
                 on] => :page
 
-    attr_reader :client, :process, :logger, :js_errors, :slowmo, :base_url,
-                :options, :window_size
+    attr_reader :client, :process, :contexts, :logger, :js_errors,
+                :slowmo, :base_url, :options, :window_size
     attr_writer :timeout
 
     def initialize(options = nil)
@@ -82,7 +82,7 @@ module Ferrum
 
     def reset
       @window_size = @original_window_size
-      targets.reset
+      contexts.reset
     end
 
     def restart
@@ -93,11 +93,7 @@ module Ferrum
     def quit
       @client.close
       @process.stop
-      @client = @process = @targets = nil
-    end
-
-    def targets
-      @targets ||= Targets.new(self)
+      @client = @process = @contexts = nil
     end
 
     def resize(**options)
@@ -115,6 +111,7 @@ module Ferrum
       Ferrum.started
       @process = Process.start(@options)
       @client = Client.new(self, @process.ws_url, 0, false)
+      @contexts = Contexts.new(self)
     end
   end
 end

data/lib/ferrum/browser/client.rb

@@ -76,7 +76,7 @@ module Ferrum
         when "Cannot find context with specified id"
           raise NoExecutionContextError.new(error)
         when "No target with given id found"
-          raise NoSuchWindowError
+          raise NoSuchPageError
         else
           raise BrowserError.new(error)
         end

data/lib/ferrum/browser/process.rb

@@ -84,6 +84,15 @@ module Ferrum
         end
       end
 
+      def self.directory_remover(path)
+        proc do
+          begin
+            FileUtils.remove_entry(path)
+          rescue Errno::ENOENT
+          end
+        end
+      end
+
       def self.detect_browser_path
         if RUBY_PLATFORM.include?("darwin")
           [
@@ -119,7 +128,9 @@ module Ferrum
         host = options.fetch(:host, BROWSER_HOST)
         @options.merge!("remote-debugging-address" => host)
 
-        @options.merge!("user-data-dir" => Dir.mktmpdir)
+        @temp_user_data_dir = Dir.mktmpdir
+        ObjectSpace.define_finalizer(self, self.class.directory_remover(@temp_user_data_dir))
+        @options.merge!("user-data-dir" => @temp_user_data_dir)
 
         @options = DEFAULT_OPTIONS.merge(@options)
 
@@ -158,8 +169,8 @@ module Ferrum
       end
 
       def stop
-        return unless @pid
-        kill
+        kill if @pid
+        remove_temp_user_data_dir if @temp_user_data_dir
         ObjectSpace.undefine_finalizer(self)
       end
 
@@ -175,6 +186,11 @@ module Ferrum
         @pid = nil
       end
 
+      def remove_temp_user_data_dir
+        self.class.directory_remover(@temp_user_data_dir).call
+        @temp_user_data_dir = nil
+      end
+
       def parse_ws_url(read_io, timeout)
         output = ""
         start = Ferrum.monotonic_time

data/lib/ferrum/context.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "ferrum/target"
+
+module Ferrum
+  class Context
+    POSITION = %i[first last].freeze
+
+    attr_reader :id, :targets
+
+    def initialize(browser, contexts, id)
+      @browser, @contexts, @id = browser, contexts, id
+      @targets = Concurrent::Hash.new
+      @pendings = Concurrent::MVar.new
+    end
+
+    def default_target
+      @default_target ||= create_target
+    end
+
+    def page
+      default_target.page
+    end
+
+    def pages
+      @targets.values.map(&:page)
+    end
+
+    # When we call `page` method on target it triggers ruby to connect to given
+    # page by WebSocket, if there are many opened windows but we need only one
+    # it makes more sense to get and connect to the needed one only which
+    # usually is the last one.
+    def windows(pos = nil, size = 1)
+      raise ArgumentError if pos && !POSITION.include?(pos)
+      windows = @targets.values.select(&:window?)
+      windows = windows.send(pos, size) if pos
+      windows.map(&:page)
+    end
+
+    def create_page
+      create_target.page
+    end
+
+    def create_target
+      target_id = @browser.command("Target.createTarget",
+                                   browserContextId: @id,
+                                   url: "about:blank")["targetId"]
+      target = @pendings.take(@browser.timeout)
+      raise NoSuchTargetError unless target.is_a?(Target)
+      @targets[target.id] = target
+      target
+    end
+
+    def add_target(params)
+      target = Target.new(@browser, params)
+      if target.window?
+        @targets[target.id] = target
+      else
+        @pendings.put(target, @browser.timeout)
+      end
+    end
+
+    def update_target(target_id, params)
+      @targets[target_id].update(params)
+    end
+
+    def delete_target(target_id)
+      @targets.delete(target_id)
+    end
+
+    def dispose
+      @contexts.dispose(@id)
+    end
+
+    def inspect
+      %(#<#{self.class} @id=#{@id.inspect} @targets=#{@targets.inspect} @default_target=#{@default_target.inspect}>)
+    end
+  end
+end

data/lib/ferrum/contexts.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "ferrum/context"
+
+module Ferrum
+  class Contexts
+    attr_reader :contexts
+
+    def initialize(browser)
+      @contexts = Concurrent::Hash.new
+      @browser = browser
+      subscribe
+      discover
+    end
+
+    def default_context
+      @default_context ||= create
+    end
+
+    def find_by(target_id:)
+      @contexts.find { |_, c| c.targets.keys.include?(target_id) }&.last
+    end
+
+    def create
+      response = @browser.command("Target.createBrowserContext")
+      context_id = response["browserContextId"]
+      context = Context.new(@browser, self, context_id)
+      @contexts[context_id] = context
+      context
+    end
+
+    def dispose(context_id)
+      context = @contexts[context_id]
+      @browser.command("Target.disposeBrowserContext",
+                       browserContextId: context.id)
+      @contexts.delete(context_id)
+      true
+    end
+
+    def reset
+      @default_context = nil
+      @contexts.keys.each { |id| dispose(id) }
+    end
+
+    private
+
+    def subscribe
+      @browser.client.on("Target.targetCreated") do |params|
+        info = params["targetInfo"]
+        next unless info["type"] == "page"
+
+        context_id = info["browserContextId"]
+        @contexts[context_id]&.add_target(info)
+      end
+
+      @browser.client.on("Target.targetInfoChanged") do |params|
+        info = params["targetInfo"]
+        next unless info["type"] == "page"
+
+        context_id, target_id = info.values_at("browserContextId", "targetId")
+        @contexts[context_id]&.update_target(target_id, info)
+      end
+
+      @browser.client.on("Target.targetDestroyed") do |params|
+        if context = find_by(target_id: params["targetId"])
+          context.delete_target(params["targetId"])
+        end
+      end
+
+      @browser.client.on("Target.targetCrashed") do |params|
+        if context = find_by(target_id: params["targetId"])
+          context.delete_target(params["targetId"])
+        end
+      end
+    end
+
+    def discover
+      @browser.command("Target.setDiscoverTargets", discover: true)
+    end
+  end
+end

data/lib/ferrum/network.rb

@@ -73,7 +73,7 @@ module Ferrum
           @authorized_ids[type] << request.interception_id
           request.continue(authChallengeResponse: response)
         elsif index + 1 < total
-          next # There are other callbacks that can handle this, skip
+          next # There are other callbacks that can handle this
         else
           request.continue
         end

data/lib/ferrum/network/request.rb

@@ -30,6 +30,10 @@ module Ferrum
         @request["url"]
       end
 
+      def url_fragment
+        @request["urlFragment"]
+      end
+
       def method
         @request["method"]
       end

data/lib/ferrum/page.rb

@@ -14,8 +14,6 @@ require "ferrum/browser/client"
 
 module Ferrum
   class Page
-    NEW_WINDOW_WAIT = ENV.fetch("FERRUM_NEW_WINDOW_WAIT", 0.3).to_f
-
     class Event < Concurrent::Event
       def iteration
         synchronize { @iteration }
@@ -37,7 +35,7 @@ module Ferrum
                 :headers, :cookies, :network,
                 :mouse, :keyboard
 
-    def initialize(target_id, browser, new_window = false)
+    def initialize(target_id, browser)
       @target_id, @browser = target_id, browser
       @event = Event.new.tap(&:set)
 
@@ -45,11 +43,6 @@ module Ferrum
       @waiting_frames ||= Set.new
       @frame_stack = []
 
-      # Dirty hack because new window doesn't have events at all
-      sleep(NEW_WINDOW_WAIT) if new_window
-
-      @session_id = @browser.command("Target.attachToTarget", targetId: @target_id)["sessionId"]
-
       host = @browser.process.host
       port = @browser.process.port
       ws_url = "ws://#{host}:#{port}/devtools/page/#{@target_id}"
@@ -83,12 +76,7 @@ module Ferrum
 
     def close
       @headers.clear
-      @browser.command("Target.detachFromTarget", sessionId: @session_id)
       @browser.command("Target.closeTarget", targetId: @target_id)
-      close_connection
-    end
-
-    def close_connection
       @client.close
     end
 
@@ -173,10 +161,6 @@ module Ferrum
         end
       end
 
-      on("Page.windowOpen") do
-        @browser.targets.refresh
-      end
-
       on("Page.navigatedWithinDocument") do
         @event.set if @waiting_frames.empty?
       end

data/lib/ferrum/page/screenshot.rb

@@ -19,6 +19,19 @@ module Ferrum
         save_file(path, data)
       end
 
+      def viewport_size
+        evaluate <<~JS
+          [window.innerWidth, window.innerHeight]
+        JS
+      end
+
+      def document_size
+        evaluate <<~JS
+          [document.documentElement.offsetWidth,
+           document.documentElement.offsetHeight]
+        JS
+      end
+
       private
 
       def save_file(path, data)
@@ -57,7 +70,7 @@ module Ferrum
         end
 
         if !!opts[:full]
-          width, height = evaluate("[document.documentElement.offsetWidth, document.documentElement.offsetHeight]")
+          width, height = document_size
           options.merge!(clip: { x: 0, y: 0, width: width, height: height, scale: scale }) if width > 0 && height > 0
         elsif opts[:selector]
           rect = evaluate("document.querySelector('#{opts[:selector]}').getBoundingClientRect()")
@@ -66,7 +79,7 @@ module Ferrum
 
         if scale != 1.0
           if !options[:clip]
-            width, height = evaluate("[document.documentElement.clientWidth, document.documentElement.clientHeight]")
+            width, height = viewport_size
             options[:clip] = { x: 0, y: 0, width: width, height: height }
           end
 

data/lib/ferrum/target.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Target
+    NEW_WINDOW_WAIT = ENV.fetch("FERRUM_NEW_WINDOW_WAIT", 0.3).to_f
+
+    def initialize(browser, params = nil)
+      @browser = browser
+      @params = params
+    end
+
+    def update(params)
+      @params = params
+    end
+
+    def page
+      @page ||= begin
+        # Dirty hack because new window doesn't have events at all
+        sleep(NEW_WINDOW_WAIT) if window?
+        Page.new(id, @browser)
+      end
+    end
+
+    def id
+      @params["targetId"]
+    end
+
+    def type
+      @params["type"]
+    end
+
+    def title
+      @params["title"]
+    end
+
+    def url
+      @params["url"]
+    end
+
+    def opener_id
+      @params["openerId"]
+    end
+
+    def context_id
+      @params["browserContextId"]
+    end
+
+    def window?
+      !!opener_id
+    end
+  end
+end

data/lib/ferrum/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ferrum
-  VERSION = "0.4"
+  VERSION = "0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ferrum
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: '0.5'
 platform: ruby
 authors:
 - Dmitry Vorotilin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-16 00:00:00.000000000 Z
+date: 2019-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: websocket-driver
@@ -170,20 +170,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
 description: Ferrum allows you to control headless Chrome browser
 email:
 - d.vorotilin@gmail.com
@@ -199,6 +185,8 @@ files:
 - lib/ferrum/browser/process.rb
 - lib/ferrum/browser/subscriber.rb
 - lib/ferrum/browser/web_socket.rb
+- lib/ferrum/context.rb
+- lib/ferrum/contexts.rb
 - lib/ferrum/cookies.rb
 - lib/ferrum/dialog.rb
 - lib/ferrum/headers.rb
@@ -217,7 +205,7 @@ files:
 - lib/ferrum/page/frame.rb
 - lib/ferrum/page/runtime.rb
 - lib/ferrum/page/screenshot.rb
-- lib/ferrum/targets.rb
+- lib/ferrum/target.rb
 - lib/ferrum/version.rb
 homepage: https://github.com/route/ferrum
 licenses:

data/lib/ferrum/targets.rb

@@ -1,123 +0,0 @@
-# frozen_string_literal: true
-
-module Ferrum
-  class Targets
-    TARGETS_RETRY_ATTEMPTS = 3
-    TARGETS_RETRY_WAIT = 0.001
-
-    def initialize(browser)
-      @page = nil
-      @mutex = Mutex.new
-      @browser = browser
-      @_default = targets.first["targetId"]
-
-      @browser.client.on("Target.detachedFromTarget") do |params|
-        page = remove_page(params["targetId"])
-        page&.close_connection
-      end
-
-      reset
-    end
-
-    def push(target_id, page = nil)
-      @targets[target_id] = page
-    end
-
-    def refresh
-      @mutex.synchronize do
-        targets.each { |t| push(t["targetId"]) if !default?(t) && !has?(t) }
-      end
-    end
-
-    def page
-      raise NoSuchWindowError unless @page
-      @page
-    end
-
-    def window_handle
-      page.target_id
-    end
-
-    def window_handles
-      @mutex.synchronize { @targets.keys }
-    end
-
-    def switch_to_window(target_id)
-      @page = find_or_create_page(target_id)
-    end
-
-    def open_new_window
-      target_id = @browser.command("Target.createTarget", url: "about:blank", browserContextId: @_context_id)["targetId"]
-      page = Page.new(target_id, @browser, true)
-      push(target_id, page)
-      target_id
-    end
-
-    def close_window(target_id)
-      remove_page(target_id)&.close
-    end
-
-    def within_window(locator)
-      original = window_handle
-
-      if window_handles.include?(locator)
-        switch_to_window(locator)
-        yield
-      else
-        raise NoSuchWindowError
-      end
-    ensure
-      switch_to_window(original)
-    end
-
-    def reset
-      if @page
-        @page.close
-        @browser.command("Target.disposeBrowserContext", browserContextId: @_context_id)
-      end
-
-      @page = nil
-      @targets = {}
-      @_context_id = nil
-
-      @_context_id = @browser.command("Target.createBrowserContext")["browserContextId"]
-      target_id = @browser.command("Target.createTarget", url: "about:blank", browserContextId: @_context_id)["targetId"]
-      @page = Page.new(target_id, @browser)
-      push(target_id, @page)
-    end
-
-    def find_or_create_page(target_id)
-      page = @targets[target_id]
-      page ||= Page.new(target_id, @browser, true)
-      @targets[target_id] ||= page
-      page
-    end
-
-    private
-
-    def remove_page(target_id)
-      page = @targets.delete(target_id)
-      @page = nil if page && @page == page
-      page
-    end
-
-    def targets
-      Ferrum.with_attempts(errors: EmptyTargetsError,
-                           max: TARGETS_RETRY_ATTEMPTS,
-                           wait: TARGETS_RETRY_WAIT) do
-        # Targets cannot be empty the must be at least one default target.
-        targets = @browser.command("Target.getTargets")["targetInfos"]
-        raise EmptyTargetsError if targets.empty?
-        targets
-      end
-    end
-
-    def default?(target)
-      @_default == target["targetId"]
-    end
-
-    def has?(target)
-      @targets.key?(target["targetId"])
-    end
-  end
-end