browsed 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4631a9e5e0982106f635f4ac515e7cf4e25259df9cbf14b40526e30fa1737147
-  data.tar.gz: 025df085a9ccdb79abb3282c20901831195e098e36182a2592dd62585e888f0f
+  metadata.gz: 1ac593ddda39f90bf445f8e858cd4a05a3b08cf997f33b1ee448f9791127148c
+  data.tar.gz: e79c1d0d006db49e81b872706868ca00d5ddc087b52738e19a863df33c598cf3
 SHA512:
-  metadata.gz: b6802e8ba20979342205b7bd4a5a111cbca200269dcbc49521a17e53a6b4743a46901d2b601defb48bcc4b9364c62b74ade6ef7dd145f80ea9e2521190355697
-  data.tar.gz: 988985ccb8ba4f478c7599e9d5d37a27e36b45446ffb82d13661d532d64a8b8869c6f16efb175e9d40dfaec8d9553116e2aa876f0656c648f54471c5b284d4ce
+  metadata.gz: 7bc94644afe6938c6df5ce3e3ad53a7c99b93acc52334fab62a1a03c0913551071a07aa3525ff400d97180bc508ac7cbdff645ca2c9e41ff66ba0913769240aa
+  data.tar.gz: 80db44c4e2bae419e34e5aba0c28d597a55d1204ce3d2c0cce1fbda40204bfd959fb0180e4b8b74a6284a4f5bd2fcad2d5536c7f24405018a6c5229e2f3c1dde

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    browsed (0.1.8)
+    browsed (0.1.9)
       agents
       capybara (~> 3.4, >= 3.4.1)
       poltergeist (~> 1.18, >= 1.18.1)

data/lib/browsed/chrome.rb

@@ -2,7 +2,7 @@ module Browsed
   module Chrome
     
     private
-      def register_chrome_driver(driver_options: {}, timeout: 60, debug: false)
+      def register_chrome_driver(options: {}, timeout: 60, debug: false)
         profile                 =   Selenium::WebDriver::Chrome::Profile.new
         
         profile["user-agent"]   =   self.user_agent unless self.user_agent.to_s.empty?

data/lib/browsed/client.rb

@@ -1,12 +1,10 @@
 module Browsed
   class Client
     attr_accessor :configuration
+    attr_accessor :driver, :browser, :browser_id, :environment
+    attr_accessor :session, :proxy
+    attr_accessor :device, :user_agent, :resolution
     attr_accessor :manager, :maximum_processes
-    attr_accessor :driver, :browser, :environment
-    attr_accessor :session
-    attr_accessor :device, :proxy
-    attr_accessor :user_agent
-    attr_accessor :resolution
     
     include Capybara::DSL
     
@@ -18,7 +16,7 @@ module Browsed
                    user_agent:        nil,
                    resolution:        nil,
                    environment:       :production,
-                   driver_options:    {},
+                   options:           {},
                    maximum_processes: nil)
       
       self.configuration                =   configuration
@@ -27,34 +25,36 @@ module Browsed
       self.browser                      =   browser           || self.configuration.browser
       self.environment                  =   environment       || self.configuration.environment
       
+      self.browser_id                   =   generate_browser_id
       self.device                       =   device
       self.proxy                        =   proxy
       
-      self.manager                      =   Browsed::Manager.new(browser: self.browser)
+      self.manager                      =   ::Browsed::Manager.new(browser: self.browser)
       self.maximum_processes            =   maximum_processes || self.configuration.maximum_processes
       
       set_user_agent(user_agent)
       set_resolution(resolution)
       
-      setup_capybara(driver_options: driver_options)
+      options.merge!(browser_id: self.browser_id)
+      setup_capybara(options: options)
     end
     
     include ::Browsed::Poltergeist
     include ::Browsed::Firefox
     include ::Browsed::Chrome
     
-    def setup_capybara(driver_options: {}, retries: 3)
+    def setup_capybara(options: {}, retries: 3)
       if can_start_new_process?
-        register_driver!(driver_options)
+        register_driver!(options)
       
         Capybara.default_driver         =   self.driver
         Capybara.javascript_driver      =   self.driver
       
-        Capybara.default_max_wait_time  =   driver_options.fetch(:wait_time, 30) #seconds
+        Capybara.default_max_wait_time  =   options.fetch(:wait_time, 30) #seconds
       
         self.session                    =   Capybara::Session.new(self.driver)
       else
-        raise Browsed::TooManyProcessesError, "Too many PhantomJS processes running, reached maximum allowed number of #{self.maximum_processes}"
+        raise Browsed::TooManyProcessesError, "Too many #{self.browser} processes running, reached maximum allowed number of #{self.maximum_processes} processes."
       end
     end
     
@@ -78,25 +78,38 @@ module Browsed
     def reset_session!
       self.session.reset_session!
     end
+    
+    def generate_browser_id
+      SecureRandom.hex[0..15]
+    end
   
-    def terminate_session!
-      self.session.reset_session!
-      self.session.driver.quit
-      self.session            =   nil
+    def quit!(retries: 3)
+      begin
+        self.session.driver.quit
+      rescue Exception
+        retries       -=    1
+        retry if retries > 0
+      end
+      
+      # If Selenium/Phantom somehow isn't able to shut down the browser, force a shutdown using kill -9
+      self.manager.set_command(browser_id: self.browser_id)
+      self.manager.kill_processes!
+      
+      self.session     =   nil
     end
     
     private
-      def register_driver!(driver_options = {})
+      def register_driver!(options = {})
         if poltergeist?
-          register_poltergeist_driver(driver_options: driver_options)
+          register_poltergeist_driver(options: options)
         elsif selenium?
           if firefox_browser?
-            register_firefox_driver(driver_options: driver_options)
+            register_firefox_driver(options: options)
           elsif firefox_headless_browser?
-            register_firefox_driver(driver_options: driver_options.merge(headless: true))
+            register_firefox_driver(options: options.merge(headless: true))
           elsif chrome_browser?
             self.driver                 =   :selenium_chrome
-            register_chrome_driver(driver_options: driver_options)
+            register_chrome_driver(options: options)
           end
         end
       end

data/lib/browsed/firefox.rb

@@ -2,10 +2,11 @@ module Browsed
   module Firefox
     
     private
-      def register_firefox_driver(driver_options: {}, timeout: 60, debug: false)
-        headless                =   driver_options.fetch(:headless, false)
-        download_path           =   driver_options.fetch(:download_path, self.configuration.download_path)
-        private_browsing        =   driver_options.fetch(:private_browsing, false)
+      def register_firefox_driver(options: {}, timeout: 60, debug: false)
+        browser_id              =   options.fetch(:browser_id, nil)
+        headless                =   options.fetch(:headless, false)
+        download_path           =   options.fetch(:download_path, self.configuration.download_path)
+        private_browsing        =   options.fetch(:private_browsing, false)
         
         profile                 =   Selenium::WebDriver::Firefox::Profile.new
         
@@ -22,15 +23,21 @@ module Browsed
           profile["pdfjs.disabled"]                           =   true
         end
         
+        profile["browser.tabs.warnOnClose"]                   =   false
+        profile["browser.tabs.warnOnCloseOtherTabs"]          =   false
         profile["general.useragent.override"]                 =   self.user_agent unless self.user_agent.to_s.empty?
         
+        id                      =   SecureRandom.hex[0..15]
         profile                 =   firefox_proxy_options(profile)
         options                 =   Selenium::WebDriver::Firefox::Options.new(profile: profile)
-        options.args           <<   "--headless" if headless
+        options.args           <<   "--browser_id=#{id}" unless browser_id.to_s.empty?
+        options.args           <<   "--headless"         if headless
         
         Capybara.register_driver self.driver do |app|
           Capybara::Selenium::Driver.new(app, browser: :firefox, options: options)
         end
+        
+        return id
       end
     
       def firefox_proxy_options(profile)

data/lib/browsed/manager.rb

@@ -1,46 +1,35 @@
 module Browsed
   class Manager
-    attr_accessor :command, :kill_signal
+    attr_accessor :command, :kill_signal, :logging
     
-    def initialize(browser: :phantomjs, kill_signal: 9)
-      case browser
-        when :phantomjs
-          self.command    =   "ps -ef | grep /[p]hantomjs"
-        when :firefox
-          self.command    =   "ps -ef | grep /[g]eckodriver"
-        when :chrome
-          self.command    =   "ps -ef | grep /[c]hromedriver"
-        else
-          self.command    =   "ps -ef | grep /[p]hantomjs"
-      end
+    def initialize(command: nil, browser_id: nil, browser: :phantomjs, kill_signal: 9, logging: false)
+      set_command(command: command, browser_id: browser_id, browser: browser)
       
       self.kill_signal    =   kill_signal
+      self.logging        =   logging
     end
     
-    def can_start_more_processes?(max_count: 5)
-      return get_current_processes.size < max_count
-    end
-    
-    def reap_stale_processes(started_after: ::Browsed.configuration.processes_max_ttl.call)
-      processes           =   get_current_processes
-      
-      processes.each do |process|
-        if (process[:date] && process[:date] < (Time.now - started_after))
-          info "[Browsed::Manager] - #{Time.now.to_s(:db)}: Process with PID #{process[:pid]} was started before #{started_after.to_s(:db)}. Process should be terminated."
-          kill_process(process)
+    def set_command(command: nil, browser_id: nil, browser: :phantomjs)
+      if !command.to_s.empty?
+        self.command        =   command
+      elsif !browser_id.to_s.empty?
+        self.command        =   "ps aux | awk '/--browser_id=#{browser_id}/'"
+      else
+        case browser
+          when :phantomjs
+            self.command    =   "ps -ef | grep /[p]hantomjs"
+          when :firefox
+            self.command    =   "ps -ef | grep /[f]irefox-bin"
+          when :chrome
+            self.command    =   "ps -ef | grep /[c]hromedriver"
+          else
+            self.command    =   "ps -ef | grep /[p]hantomjs"
         end
-      end if processes && processes.any?
+      end
     end
     
-    def kill_process(process)
-      info "[Browsed::Manager] - #{Time.now.to_s(:db)}: Killing process with PID #{process[:pid]} matching command #{self.command}."
-      
-      begin
-        ::Process.kill(self.kill_signal, process[:pid])
-      
-      rescue StandardError => e
-        info "[Browsed::Manager] - #{Time.now.to_s(:db)}:  Failed to kill process with pid '#{process[:pid]}'. Error Class: #{e.class.name}. Error Message: #{e.message}"
-      end
+    def can_start_more_processes?(max_count: nil)
+      return max_count.nil? || get_current_processes.size < max_count
     end
     
     def get_current_processes
@@ -55,44 +44,63 @@ module Browsed
       return processes
     end
     
-    private
+    def kill_processes!(started_after: nil)
+      processes           =   get_current_processes
+      
+      processes.each do |process|
+        killable          =   started_after.nil? || (process[:date] && process[:date] < (Time.now - started_after))
+        kill_process!(process) if killable
+      end if processes && processes.any?
+    end
     
-    def parse_process(process_data)
-      process             =   {}
+    def kill_process!(process)
+      info "[Browsed::Manager] - #{Time.now.to_s}: Killing process with PID #{process[:pid]} matching command #{self.command}."
       
-      parts               =   process_data.split(' ')
-      pid                 =   parts[1].to_i
-      started             =   parts[4].to_s
-      date                =   parse_date(started)
+      begin
+        ::Process.kill(self.kill_signal, process[:pid])
+      
+      rescue StandardError => e
+        info "[Browsed::Manager] - #{Time.now.to_s}:  Failed to kill process with pid '#{process[:pid]}'. Error Class: #{e.class.name}. Error Message: #{e.message}"
+      end
+    end
+    
+    private
+      def parse_process(process_data)
+        process             =   {}
       
-      process[:pid]       =   pid
-      process[:started]   =   started
-      process[:date]      =   date
+        parts               =   process_data.split(' ')
+        pid                 =   parts[1].to_i
+        started             =   parts[4].to_s
+        date                =   parse_date(started)
       
-      info "[Browsed::Manager] - #{Time.now.to_s(:db)}: Pid: #{pid}. Started: #{started}. Date: #{date}.\n"
+        process[:pid]       =   pid
+        process[:started]   =   started
+        process[:date]      =   date
       
-      return process
-    end
+        info "[Browsed::Manager] - #{Time.now.to_s}: Pid: #{pid}. Started: #{started}. Date: #{date}.\n"
+      
+        return process
+      end
     
-    def parse_date(date, retries = 3)
-      begin
-        if (!(date =~ /^[a-z]{3,4}\d*/i).nil?) #Sep16
-          parsed_date = DateTime.strptime(date, "%b%d")
+      def parse_date(date, retries = 3)
+        begin
+          if (!(date =~ /^[a-z]{3,4}\d*/i).nil?) #Sep16
+            parsed_date = DateTime.strptime(date, "%b%d")
         
-        elsif (!(date =~ /^\d*:\d*/i).nil?) #11:34
-          parsed_date = Time.strptime(date, "%H:%M").to_datetime
-        end
+          elsif (!(date =~ /^\d*:\d*/i).nil?) #11:34
+            parsed_date = Time.strptime(date, "%H:%M").to_datetime
+          end
       
-      rescue StandardError => e
-        info "[Browsed::Manager] - #{Time.now.to_s(:db)}: Exception occurred while trying to parse date/time string '#{date}'. Error Class: #{e.class.name}. Error: #{e.message}."
-        retries -= 1
-        retry if retries > 0
+        rescue StandardError => e
+          info "[Browsed::Manager] - #{Time.now.to_s}: Exception occurred while trying to parse date/time string '#{date}'. Error Class: #{e.class.name}. Error: #{e.message}."
+          retries -= 1
+          retry if retries > 0
+        end
       end
-    end
     
-    def info(message, enabled = true)
-      puts message if enabled
-    end
+      def info(message)
+        puts message if self.logging
+      end
     
   end
 end

data/lib/browsed/poltergeist.rb

@@ -2,9 +2,9 @@ module Browsed
   module Poltergeist
     
     private
-      def register_poltergeist_driver(driver_options: {}, timeout: 60, debug: false)
+      def register_poltergeist_driver(options: {}, timeout: 60, debug: false)
         phantom_opts            =   ['--ignore-ssl-errors=true', '--ssl-protocol=any']
-        disable_images          =   driver_options.fetch(:disable_images, false)
+        disable_images          =   options.fetch(:disable_images, false)
     
         if disable_images
           phantom_opts         <<   "--load-images=false"

data/lib/browsed/version.rb

@@ -1,3 +1,3 @@
 module Browsed
-  VERSION = "0.1.8"
+  VERSION = "0.1.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: browsed
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Sebastian
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-26 00:00:00.000000000 Z
+date: 2018-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara