browsed 0.2.5 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 343ac6bd7d2660fa5503615f384933df9dae903a69d242791edf03ba75d93fea
-  data.tar.gz: aa8d6d88db9dd61ae94fb4bc60f1b30c60923239209e1732cc7613c14e97accd
+  metadata.gz: 4ce77ce8ab4a14169daeb61e3af46f43340b248d84c81cdda763777e7c152bea
+  data.tar.gz: 2694ba144b2f55c5a7e1dd8231c258092828794b3d5f229edf126814aa2c176f
 SHA512:
-  metadata.gz: 20439f0110dac3a95d059f47feedc6d2a26d15b7bf3830f5d633307100aa0e6d0e0ade531dd46692dcfd02ffdd69d8f8e241818310a1bbf1f6fee520ee5143e6
-  data.tar.gz: 3002d0d9c1c76be747c03d5e6f162fd97e7c51e1b8f1bc0ad97c5224a812cd51a821a4ac314f8b7be5219c8dcce0948b44b46b6cd5ad0e6f3881399e8e295b6b
+  metadata.gz: 80c7d784ad587b6202e2c4cec78830793e7f1101f10a8c6c49c2bce6c47e556b4019ac6fc616ebda318fce632834f5fd7fd10d85080c2fb3cfa9ce9091d933df
+  data.tar.gz: 4ec3b57515d439944e62ea0c791e92c4c97c64600b7b47342bc81dd7e9b02fd1c63aac0a5989931950c9862e15621c936d49e21ae98b2d8d5f6d5c0db401176e

data/.gitignore

@@ -9,6 +9,8 @@
 /tmp/
 /files/
 
+proxies.yml
+
 # rspec failure tracking
 .rspec_status
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    browsed (0.2.4)
+    browsed (0.2.5)
       agents (>= 0.1.4)
       capybara (>= 2.18)
       poltergeist (>= 1.18)
@@ -29,7 +29,7 @@ GEM
     launchy (2.4.3)
       addressable (~> 2.3)
     method_source (0.9.2)
-    mini_mime (1.0.1)
+    mini_mime (1.0.2)
     mini_portile2 (2.4.0)
     nokogiri (1.10.3)
       mini_portile2 (~> 2.4.0)

data/README.md

@@ -1,6 +1,6 @@
 # Browsed
 
-Browsed is a lightweight Capybara/PhantomJS/Selenium framework with tools and utilities for randomizing user agents, resolutions etc.
+Browsed is a wrapper around Capybara/Selenium with tools for proxy management and utilities for randomizing user agents, resolutions etc.
 
 ## Installation
 
@@ -21,31 +21,26 @@ Or install it yourself as:
 ## Usage
 
 ```ruby
-# driver can be :poltergeist (PhantomJS) or :selenium/:selenium_chrome/:selenium_chrome_headless (Firefox/Chrome)
-# browser can be: :phantomjs, :chrome, :chrome_headless, :firefox, :firefox_headless
+# browser can be: :chrome, :firefox or :phantomjs (PhantomJS might be deprecated eventually though)
 # device can be :desktop, :phone (iphone/android), :tablet (ipad/android), :iphone, :ipad, :android_phone, :android_tablet
-client    =   Browsed::Client.new(driver: :selenium_chrome, browser: :chrome, device: :desktop)
+client    =   Browsed::Client.new(browser: :chrome, headless: false, device: :desktop)
+client.session.visit "https://www.google.com"
 ```
 
-If you want to use proxies (note that proxy authentication is only possible using Poltergeist/PhantomJS):
+Proxy auth support is supported using two different methods:
 
-```ruby
-proxy     =   {host: "127.0.0.1", port: 8080, username: "foo", password: "bar"}
-client    =   Browsed::Client.new(driver: :poltergeist, browser: :phantomjs, device: :desktop, proxy: proxy)
-```
+1) For non-headless Google Chrome a plugin will be generated on the fly and used by Chrome to auto-fill username/password for the proxy. Major downside with this approach is that Chrome can't be run in headless mode.
+
+2) Using the NodeJS library proxy-chain. Browsed will spawn a Node sub-process that will run a local proxy server that forwards all packets to the final proxy server. When the client is finished (by issuing client.quit!) the proxy-chain server will also be terminated. Major downside: depends on Node being installed.
 
-Use the session property to interact with the underlying Capybara::Session object:
+Here's an example using a proxy via proxy-chain:
 
 ```ruby
-client.session.visit("https://www.google.com")
+proxy     =   {host: "127.0.0.1", port: 8080, username: "foo", password: "bar", mode: :proxy_chain}
+client    =   Browsed::Client.new(browser: :chrome, headless: false, device: :desktop, proxy: proxy)
+client.session.visit "https://www.google.com"
 ```
 
-## Support
-
-Proxy authentication is currently only supported by PhantomJS and Chrome not running in headless mode.
-
-Chrome proxy authentication support is somewhat of a hack and works by dynamically generating a plugin on the fly and then starting Chrome with that plugin. Unfortunately this doesn't work in headless Chrome.
-
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/bin/browse

@@ -3,24 +3,27 @@
 require "bundler/setup"
 require "browsed"
 require "launchy"
+require "yaml"
 
-driver    =   :selenium_chrome
-browser   =   :chrome
+url         =   "https://whatismyipaddress.com"
+screenshot  =   File.expand_path("../../files/#{Time.now.to_i}.png", __FILE__)
 
-url       =   "https://whatismyipaddress.com"
-path      =   File.expand_path("../../files/#{Time.now.to_i}.png", __FILE__)
+path        =   File.join(File.dirname(__FILE__), "../proxies.yml")
+proxy       =   ::File.exists?(path) ? YAML.load_file(path).symbolize_keys : nil
 
-client    =   Browsed::Client.new(
-  driver: driver,
-  browser: browser,
+client      =   Browsed::Client.new(
+  browser: :chrome,
+  headless: true,
   device: :desktop,
   environment: :development,
   user_agent: :randomize,
   resolution: [1920,1080],
-  options: {open_timeout: 30, read_timeout: 30}
+  options: {open_timeout: 30, read_timeout: 30},
+  proxy: proxy
 )
 
 client.resize! # Set resolution
 client.session.visit(url)
-client.session.save_screenshot(path)
-client.display_screenshot!(path)
+client.session.save_screenshot(screenshot)
+client.display_screenshot!(screenshot)
+client.quit!

data/lib/browsed.rb

@@ -14,13 +14,20 @@ require "browsed/configuration"
 
 require "browsed/manager"
 
-require "browsed/extensions/chrome/proxy_authentication"
+require "browsed/proxies/chrome/proxy_authentication"
+require "browsed/proxies/proxy_chain"
 
-require "browsed/poltergeist"
+require "browsed/proxies"
+require "browsed/browser"
+require "browsed/phantomjs"
 require "browsed/firefox"
 require "browsed/chrome"
 require "browsed/client"
 
+if !Hash.instance_methods(false).include?(:symbolize_keys)
+  require "browsed/extensions/hash"
+end
+
 module Browsed
   
   class << self

data/lib/browsed/browser.rb

@@ -0,0 +1,82 @@
+module Browsed
+  module Browser
+    
+    def generate_browser_id
+      SecureRandom.hex
+    end
+    
+    # Resize the window separately and not based on initialization
+    def resize!(res = nil)
+      res ||= self.resolution
+      
+      if res && res.size.eql?(2) && resizable_browser? # Resolutions for Chrome & Poltergeist are set in the driver
+        self.session.current_window.resize_to(res.first, res.last) # [width, height]
+      end
+    end
+    
+    def resizable_browser?
+      non_resizable   =   [:poltergeist, :selenium_chrome, :selenium_chrome_headless]
+      !non_resizable.include?(self.driver.to_sym)
+    end
+    
+    # User Agents
+    def set_user_agent(user_agent)
+      if !user_agent.to_s.empty? && !user_agent.to_sym.eql?(:randomize)
+        self.user_agent       =   user_agent
+      elsif (!user_agent.to_s.empty? && user_agent.to_sym.eql?(:randomize)) || phantomjs?
+        case self.device
+          when :iphone
+            self.user_agent   =   Agents.random_user_agent(:phones, :iphone)
+          when :android_phone
+            self.user_agent   =   Agents.random_user_agent(:phones, :android)
+          when :ipad
+            self.user_agent   =   Agents.random_user_agent(:tablets, :ipad)
+          when :android_tablet
+            self.user_agent   =   Agents.random_user_agent(:tablets, :android)
+          else
+            self.user_agent   =   Agents.random_user_agent(self.device)
+        end
+      end
+    end
+    
+    def runs_ios?
+      Agents.runs_ios?(self.user_agent)
+    end
+  
+    def is_iphone?
+      Agents.is_iphone?(self.user_agent)
+    end
+  
+    def is_ipad?
+      Agents.is_ipad?(self.user_agent)
+    end
+    
+    # Resolution      
+    def set_resolution(res)
+      if res && res.is_a?(Array)
+        self.resolution   =   res
+      elsif res && res.is_a?(Symbol) && res.eql?(:randomize)
+        self.resolution   =   randomize_resolution
+      end
+    end
+    
+    def randomize_resolution
+      runs_ios? ? randomize_ios_resolution : Browsed::Constants::RESOLUTIONS.fetch(self.device, :desktop).sample
+    end
+  
+    def randomize_ios_resolution
+      resolution_device =   case self.device
+        when :iphone, :android_phone
+          :phone
+        when :ipad, :android_tablet
+          :tablet
+        else
+          self.device
+      end
+      
+      random_key    =   Browsed::Constants::RESOLUTIONS.fetch(resolution_device, :desktop).keys.sample
+      resolution    =   Browsed::Constants::RESOLUTIONS.fetch(resolution_device, :desktop)[random_key]
+    end
+    
+  end
+end

data/lib/browsed/chrome.rb

@@ -29,21 +29,30 @@ module Browsed
       end
     
       def manage_chrome_proxy
-        proxy_options           =   nil
-        plugin_path             =   nil
+        proxy_options                 =   nil
+        plugin_path                   =   nil
         
-        if self.proxy && !self.proxy.empty? && self.proxy.has_key?(:host) && self.proxy.has_key?(:port)
-          proxy_options         =   Selenium::WebDriver::Proxy.new(
-            http: "#{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)}",
-            ssl:  "#{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)}"
-          )
+        if valid_proxy?
+          if self.proxy.fetch(:mode, nil).eql?(:proxy_chain) && proxy_using_auth?
+            self.proxy_chain_server   =   Browsed::Proxies::ProxyChain.new(self.browser_id)
+            log("Starting a new proxy chain server instance.")
+            
+            generated_proxy_url       =   self.proxy_chain_server.start_server(generate_proxy_auth_url)
+            
+            log("Started a new proxy chain server instance at #{generated_proxy_url}.")
+            
+            uri                       =   URI.parse(generated_proxy_url)
+            proxy_options             =   generate_selenium_webdriver_proxy(host: uri.host, port: uri.port)
+            
+          else
+            proxy_options             =   generate_selenium_webdriver_proxy(host: self.proxy.fetch(:host), port: self.proxy.fetch(:port))
           
-          log("Will use proxy #{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)} to initiate the request.")
+            log("Will use proxy #{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)} to initiate the request.")
           
-          if !headless? && !self.proxy[:username].to_s.empty? && !self.proxy[:password].to_s.empty?
-            log("Generating a new proxy plugin to manage proxy authentication.")
-            
-            plugin_path         =   Browser::Extensions::Chrome::ProxyAuthentication::Packager.package_extension(proxy, self.configuration.temp_path)
+            if !headless? && proxy_using_auth?
+              log("Generating a new proxy plugin to manage proxy authentication.")
+              plugin_path             =   Browser::Proxies::Chrome::ProxyAuthentication::Packager.package_extension(proxy, self.configuration.temp_path)
+            end
           end
         end
       

data/lib/browsed/client.rb

@@ -1,8 +1,8 @@
 module Browsed
   class Client
     attr_accessor :configuration
-    attr_accessor :driver, :browser, :browser_id, :environment
-    attr_accessor :session, :proxy
+    attr_accessor :driver, :browser, :browser_id, :headless, :environment
+    attr_accessor :session, :proxy, :proxy_chain_server
     attr_accessor :device, :user_agent, :resolution, :headless
     attr_accessor :manager, :maximum_processes
     
@@ -11,6 +11,7 @@ module Browsed
     def initialize(configuration:     ::Browsed.configuration,
                    driver:            :selenium_chrome,
                    browser:           :chrome,
+                   headless:          false,
                    device:            :desktop,
                    proxy:             nil,
                    user_agent:        nil,
@@ -19,20 +20,20 @@ module Browsed
                    options:           {},
                    maximum_processes: nil)
       
-      self.configuration                =   configuration
+      self.configuration        =   configuration
+      self.headless             =   headless
+      self.environment          =   environment || self.configuration.environment
       
-      self.driver                       =   driver            || self.configuration.driver
-      self.browser                      =   browser           || self.configuration.browser
-      self.environment                  =   environment       || self.configuration.environment
+      self.device               =   device
       
-      self.headless                     =   false
-      self.browser_id                   =   generate_browser_id
-      self.device                       =   device
-      self.proxy                        =   proxy
+      self.proxy                =   proxy
+      self.proxy_chain_server   =   nil
       
-      self.manager                      =   ::Browsed::Manager.new(browser: self.browser)
-      self.maximum_processes            =   maximum_processes || self.configuration.maximum_processes
+      self.manager              =   ::Browsed::Manager.new(browser: self.browser)
+      self.maximum_processes    =   maximum_processes || self.configuration.maximum_processes
       
+      set_browser(browser)
+      set_driver
       set_user_agent(user_agent)
       set_resolution(resolution)
       
@@ -40,9 +41,29 @@ module Browsed
       setup_capybara(options: options)
     end
     
-    include ::Browsed::Poltergeist
+    include ::Browsed::Browser
+    include ::Browsed::Proxies
     include ::Browsed::Firefox
     include ::Browsed::Chrome
+    include ::Browsed::Phantomjs
+    
+    def set_driver
+      self.driver   =   case self.browser.to_sym
+        when :chrome
+          self.headless ? :selenium_chrome_headless : :selenium_chrome
+        when :firefox
+          :selenium
+        when :phantomjs
+          :poltergeist
+      end
+    end
+    
+    def set_browser(browser)
+      self.browser      =   (browser || self.configuration.browser)&.to_sym
+      self.browser_id   =   generate_browser_id
+      
+      raise ::Browsed::Errors::InvalidBrowserError unless ::Browsed::Constants::BROWSERS.include?(self.browser)
+    end
     
     def setup_capybara(options: {}, retries: 3)
       self.manager.kill_stale_processes!
@@ -69,27 +90,9 @@ module Browsed
       Launchy.open path if development?
     end
     
-    # Resize the window separately and not based on initialization
-    def resize!(res = nil)
-      res ||= self.resolution
-      
-      if res && res.size.eql?(2) && resizable_browser?(self.driver) # Resolutions for Chrome & Poltergeist are set in the driver
-        self.session.current_window.resize_to(res.first, res.last) # [width, height]
-      end
-    end
-    
-    def resizable_browser?(driver)
-      non_resizable = [:poltergeist, :selenium_chrome, :selenium_chrome_headless]
-      !non_resizable.include?(driver.to_sym)
-    end
-    
     def reset_session!
       self.session.reset_session!
     end
-    
-    def generate_browser_id
-      SecureRandom.hex[0..15]
-    end
   
     def quit!(retries: 3)
       begin
@@ -103,52 +106,37 @@ module Browsed
       self.manager.set_command(browser_id: self.browser_id)
       self.manager.kill_processes!
       
-      self.session     =   nil
+      if self.proxy_chain_server
+        self.proxy_chain_server.stop_server
+        self.proxy_chain_server   =   nil
+      end
+      
+      self.session                =   nil
     end
     
     private
       def register_driver!(options = {})
-        if poltergeist?
-          register_poltergeist_driver(options: options)
-        elsif firefox_browser?
-          register_firefox_driver(options: options)
-        elsif firefox_headless_browser?
-          self.headless               =   true
+        if firefox?
           register_firefox_driver(options: options)
-        elsif chrome_browser?
-          self.driver                 =   :selenium_chrome
-          register_chrome_driver(options: options.merge(resolution: self.resolution))
-        elsif chrome_headless_browser?
-          self.headless               =   true
-          self.driver                 =   :selenium_chrome_headless
+        elsif chrome?
           register_chrome_driver(options: options.merge(resolution: self.resolution))
+        elsif phantomjs?
+          register_phantomjs_driver(options: options)
         end
       end
     
-      def poltergeist?
-        self.driver.to_sym.eql?(:poltergeist)
+      def phantomjs?
+        self.browser.to_sym.eql?(:phantomjs)
       end
       
-      def selenium?
-        self.driver.to_sym.eql?(:selenium)
-      end
-      
-      def firefox_browser?
+      def firefox?
         self.browser.to_sym.eql?(:firefox)
       end
       
-      def firefox_headless_browser?
-        self.browser.to_sym.eql?(:firefox_headless)
-      end
-      
-      def chrome_browser?
+      def chrome?
         self.browser.to_sym.eql?(:chrome)
       end
       
-      def chrome_headless_browser?
-        self.browser.to_sym.eql?(:chrome_headless)
-      end
-      
       def development?
         in_environment?(:development)
       end
@@ -160,76 +148,6 @@ module Browsed
       def headless?
         self.headless
       end
-    
-      # User Agents
-      def set_user_agent(user_agent)
-        if !user_agent.to_s.empty? && !user_agent.to_sym.eql?(:randomize)
-          self.user_agent       =   user_agent
-        elsif (!user_agent.to_s.empty? && user_agent.to_sym.eql?(:randomize)) || poltergeist?
-          case self.device
-            when :iphone
-              self.user_agent   =   Agents.random_user_agent(:phones, :iphone)
-            when :android_phone
-              self.user_agent   =   Agents.random_user_agent(:phones, :android)
-            when :ipad
-              self.user_agent   =   Agents.random_user_agent(:tablets, :ipad)
-            when :android_tablet
-              self.user_agent   =   Agents.random_user_agent(:tablets, :android)
-            else
-              self.user_agent   =   Agents.random_user_agent(self.device)
-          end
-        end
-      end
-      
-      def runs_ios?
-        Agents.runs_ios?(self.user_agent)
-      end
-    
-      def is_iphone?
-        Agents.is_iphone?(self.user_agent)
-      end
-    
-      def is_ipad?
-        Agents.is_ipad?(self.user_agent)
-      end
-      
-      # Resolution      
-      def set_resolution(res)
-        if res && res.is_a?(Array)
-          self.resolution   =   res
-        elsif res && res.is_a?(Symbol) && res.eql?(:randomize)
-          self.resolution   =   randomize_resolution
-        end
-      end
-      
-      def randomize_resolution
-        runs_ios? ? randomize_ios_resolution : Browsed::Constants::RESOLUTIONS.fetch(self.device, :desktop).sample
-      end
-    
-      def randomize_ios_resolution
-        resolution_device =   case self.device
-          when :iphone, :android_phone
-            :phone
-          when :ipad, :android_tablet
-            :tablet
-          else
-            self.device
-        end
-        
-        random_key        =   Browsed::Constants::RESOLUTIONS.fetch(resolution_device, :desktop).keys.sample
-        resolution        =   Browsed::Constants::RESOLUTIONS.fetch(resolution_device, :desktop)[random_key]
-      end
-      
-      def wait_for_ajax
-        Timeout.timeout(Capybara.default_max_wait_time) do
-          loop until finished_all_ajax_requests?
-        end
-      end
-
-      def finished_all_ajax_requests?
-        evald             =   self.session.evaluate_script('jQuery.active')
-        evald.nil? || evald.zero?
-      end
       
       def log(message)
         puts "[Browsed::Client] - #{Time.now}: #{message}" if self.configuration.verbose?