browsed 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61e83765b6257c39cd97be1c3764a20bb20e90b6ce602c94d693b0fb4444bf64
-  data.tar.gz: 8f747af9a52257cd1a05aecfddbd81fc714e76253472eba5ef7724a6dce1e10f
+  metadata.gz: f711eb863313bb4ebb804b8a8d531304b76d9faea6afb4ad347fb418acd267b3
+  data.tar.gz: f5f7016f636a77f68519aa7db4847a3989b3f44e9b9c59326e3acbd0976b06e9
 SHA512:
-  metadata.gz: 666a4dd2ace58b39da9b7132493c0f64dc605fb50de3db3363b21f18caaa643be3cc33321c6ce5e94c372735e998e8ba611262a683591351f3512617f125ddee
-  data.tar.gz: 4b191391235827932b9b8e16942802757424bfb77f25b0a9076243ba8fa520f9da87a3faf293c29e321be0c8e64503d375fd5901344c480e5e744b8739837399
+  metadata.gz: eb714bc6d80b581185835d26d96d7ae3b1de0b673d0b9ef071f26a74dd4a15066bfaf6e7269b33e005ad4fe0197b90a8345b4466cc03d3dd807d648ed4c14df8
+  data.tar.gz: b4744b3813485365e683a4fa0a1d131ee0fb2032319b2e01dc466d413060bac1623ef0189c90db80855fe63cfd6146e8c133d2b6dda3bf8ff30bb8eede33a736

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    browsed (0.2.1)
+    browsed (0.2.4)
       agents (>= 0.1.4)
       capybara (>= 2.18)
       poltergeist (>= 1.18)
@@ -13,26 +13,25 @@ GEM
     addressable (2.6.0)
       public_suffix (>= 2.0.2, < 4.0)
     agents (0.1.4)
-    capybara (3.15.0)
+    capybara (3.25.0)
       addressable
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
-      regexp_parser (~> 1.2)
+      regexp_parser (~> 1.5)
       xpath (~> 3.2)
-    childprocess (0.9.0)
-      ffi (~> 1.0, >= 1.0.11)
+    childprocess (1.0.1)
+      rake (< 13.0)
     cliver (0.3.2)
     coderay (1.1.2)
     diff-lcs (1.3)
-    ffi (1.10.0)
     launchy (2.4.3)
       addressable (~> 2.3)
     method_source (0.9.2)
     mini_mime (1.0.1)
     mini_portile2 (2.4.0)
-    nokogiri (1.10.2)
+    nokogiri (1.10.3)
       mini_portile2 (~> 2.4.0)
     poltergeist (1.18.1)
       capybara (>= 2.1, < 4)
@@ -41,32 +40,32 @@ GEM
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    public_suffix (3.0.3)
-    rack (2.0.6)
+    public_suffix (3.1.1)
+    rack (2.0.7)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rake (12.3.2)
-    regexp_parser (1.3.0)
+    regexp_parser (1.5.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
+    rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    rubyzip (1.2.2)
-    selenium-webdriver (3.141.0)
-      childprocess (~> 0.5)
+    rspec-support (3.8.2)
+    rubyzip (1.2.3)
+    selenium-webdriver (3.142.3)
+      childprocess (>= 0.5, < 2.0)
       rubyzip (~> 1.2, >= 1.2.2)
-    websocket-driver (0.7.0)
+    websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.3)
+    websocket-extensions (0.1.4)
     xpath (3.2.0)
       nokogiri (~> 1.8)
 
@@ -75,11 +74,11 @@ PLATFORMS
 
 DEPENDENCIES
   browsed!
-  bundler (~> 2.0.1)
+  bundler (~> 1.17)
   launchy (~> 2.4, >= 2.4.3)
   pry (~> 0.12.2)
   rake (~> 12.3.2)
   rspec (~> 3.8.0)
 
 BUNDLED WITH
-   2.0.1
+   1.17.2

data/README.md

@@ -40,6 +40,12 @@ Use the session property to interact with the underlying Capybara::Session objec
 client.session.visit("https://www.google.com")
 ```
 
+## Support
+
+Proxy authentication is currently only supported by PhantomJS and Chrome not running in headless mode.
+
+Chrome proxy authentication support is somewhat of a hack and works by dynamically generating a plugin on the fly and then starting Chrome with that plugin. Unfortunately this doesn't work in headless Chrome.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/browsed.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   
   spec.add_dependency "agents", '>= 0.1.4'
 
-  spec.add_development_dependency "bundler", "~> 2.0.1"
+  spec.add_development_dependency "bundler", "~> 1.17"
   spec.add_development_dependency "rake", "~> 12.3.2"
   spec.add_development_dependency "rspec", "~> 3.8.0"
   spec.add_development_dependency "pry", "~> 0.12.2"

data/lib/browsed.rb

@@ -14,6 +14,8 @@ require "browsed/configuration"
 
 require "browsed/manager"
 
+require "browsed/extensions/chrome/proxy_authentication"
+
 require "browsed/poltergeist"
 require "browsed/firefox"
 require "browsed/chrome"

data/lib/browsed/chrome.rb

@@ -9,11 +9,15 @@ module Browsed
         
         profile["user-agent"]   =   self.user_agent unless self.user_agent.to_s.empty?
         
-        proxy_options           =   chrome_proxy_options
-        capabilities            =   !proxy_options.nil? ? Selenium::WebDriver::Remote::Capabilities.chrome(proxy: proxy_options) : {}
+        proxy_options           =   manage_chrome_proxy
+        proxy_capabilities      =   proxy_options[:capabilities]
+        capabilities            =   !proxy_capabilities.nil? ? Selenium::WebDriver::Remote::Capabilities.chrome(proxy: proxy_capabilities) : {}
+        proxy_plugin_path       =   proxy_options[:plugin_path]
+        
         args                    =   %w(disable-popup-blocking)
         args                    =   args | %W(window-size=#{resolution.first},#{resolution.last}) if resolution && resolution.any? && resolution.size.eql?(2)
         args                    =   args | %w(headless disable-gpu) if headless
+        args                    =   args | %W(load-extension=#{proxy_plugin_path}) unless proxy_plugin_path.to_s.empty?
         
         options                 =   Selenium::WebDriver::Chrome::Options.new(profile: profile, args: args)
         
@@ -22,8 +26,9 @@ module Browsed
         end
       end
     
-      def chrome_proxy_options
+      def manage_chrome_proxy
         proxy_options           =   nil
+        plugin_path             =   nil
         
         if self.proxy && !self.proxy.empty? && self.proxy.has_key?(:host) && self.proxy.has_key?(:port)
           proxy_options         =   Selenium::WebDriver::Proxy.new(
@@ -32,9 +37,15 @@ module Browsed
           )
           
           log("Will use proxy #{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)} to initiate the request.")
+          
+          if !headless? && !self.proxy[:username].to_s.empty? && !self.proxy[:password].to_s.empty?
+            log("Generating a new proxy plugin to manage proxy authentication.")
+            
+            plugin_path         =   Browser::Extensions::Chrome::ProxyAuthentication::Packager.package_extension(proxy, self.configuration.temp_path)
+          end
         end
       
-        return proxy_options
+        return {capabilities: proxy_options, plugin_path: plugin_path}
       end
   
   end

data/lib/browsed/client.rb

@@ -3,7 +3,7 @@ module Browsed
     attr_accessor :configuration
     attr_accessor :driver, :browser, :browser_id, :environment
     attr_accessor :session, :proxy
-    attr_accessor :device, :user_agent, :resolution
+    attr_accessor :device, :user_agent, :resolution, :headless
     attr_accessor :manager, :maximum_processes
     
     include Capybara::DSL
@@ -25,6 +25,7 @@ module Browsed
       self.browser                      =   browser           || self.configuration.browser
       self.environment                  =   environment       || self.configuration.environment
       
+      self.headless                     =   false
       self.browser_id                   =   generate_browser_id
       self.device                       =   device
       self.proxy                        =   proxy
@@ -112,11 +113,13 @@ module Browsed
         elsif firefox_browser?
           register_firefox_driver(options: options)
         elsif firefox_headless_browser?
+          self.headless               =   true
           register_firefox_driver(options: options.merge(headless: true))
         elsif chrome_browser?
           self.driver                 =   :selenium_chrome
           register_chrome_driver(options: options.merge(resolution: self.resolution))
         elsif chrome_headless_browser?
+          self.headless               =   true
           self.driver                 =   :selenium_chrome_headless
           register_chrome_driver(options: options.merge(resolution: self.resolution, headless: true))
         end
@@ -153,6 +156,10 @@ module Browsed
       def in_environment?(env)
         self.environment.eql?(env)
       end
+      
+      def headless?
+        self.headless
+      end
     
       # User Agents
       def set_user_agent(user_agent)

data/lib/browsed/configuration.rb

@@ -1,7 +1,7 @@
 module Browsed
   class Configuration
     attr_accessor :driver, :browser, :environment
-    attr_accessor :binary_path, :download_path
+    attr_accessor :binary_path, :root_path, :download_path, :temp_path
     attr_accessor :maximum_processes, :processes_max_ttl
     attr_accessor :verbose
   
@@ -13,7 +13,9 @@ module Browsed
       
       self.binary_path          =   "/usr/local/bin/phantomjs"
       
+      self.root_path            =   File.expand_path(File.join(__FILE__, "../../.."))
       self.download_path        =   nil
+      self.temp_path            =   "/tmp"
       
       self.maximum_processes    =   nil
       self.processes_max_ttl    =   60 * 30 # 30 minutes

data/lib/browsed/extensions/chrome/proxy_authentication.rb

@@ -0,0 +1,78 @@
+module Browser
+  module Extensions
+    module Chrome
+      module ProxyAuthentication
+                
+        class Packager          
+          def self.package_extension(proxy, path)
+            file_id         =   Digest::SHA1.hexdigest("#{proxy[:username]}:#{proxy[:password]}@#{proxy[:host]}:#{proxy[:port]}")
+            plugin_path     =   "#{path}/chrome-proxy-authentication-plugin-#{file_id}"
+            FileUtils.mkdir_p plugin_path
+            
+            manifest_file   =   "#{plugin_path}/manifest.json"
+            File.open(manifest_file, 'w') { |file| file.write(::Browser::Extensions::Chrome::ProxyAuthentication::MANIFEST_JSON_TEMPLATE) }
+            
+            script_file     =   "#{plugin_path}/background.js"
+            script_result   =   Browser::Extensions::Chrome::ProxyAuthentication::BACKGROUND_SCRIPT_TEMPLATE % [proxy[:host], proxy[:port], proxy[:username], proxy[:password]]
+            File.open(script_file, 'w') { |file| file.write(script_result) }
+            
+            return plugin_path
+          end
+        end
+        
+        MANIFEST_JSON_TEMPLATE =  <<-TEMPLATE
+{
+  "version": "1.0.0",
+  "manifest_version": 2,
+  "name": "Chrome Proxy",
+  "permissions": [
+    "proxy",
+    "tabs",
+    "unlimitedStorage",
+    "storage",
+    "<all_urls>",
+    "webRequest",
+    "webRequestBlocking"
+  ],
+  "background": {
+    "scripts": ["background.js"]
+  },
+  "minimum_chrome_version":"22.0.0"
+}
+        TEMPLATE
+        
+        BACKGROUND_SCRIPT_TEMPLATE = <<-TEMPLATE
+var config = {
+  mode: "fixed_servers",
+  rules: {
+    singleProxy: {
+      scheme: "http",
+      host: "%s",
+      port: parseInt(%s)
+    },
+    bypassList: ["localhost"]
+  }
+};
+
+chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
+
+function callbackFn(details) {
+  return {
+    authCredentials: {
+      username: "%s",
+      password: "%s"
+    }
+  };
+}
+
+chrome.webRequest.onAuthRequired.addListener(
+  callbackFn,
+  {urls: ["<all_urls>"]},
+  ['blocking']
+);
+        TEMPLATE
+        
+      end
+    end
+  end
+end

data/lib/browsed/firefox.rb

@@ -42,10 +42,16 @@ module Browsed
     
       def firefox_proxy_options(profile)
         if self.proxy && !self.proxy.empty? && self.proxy.has_key?(:host) && self.proxy.has_key?(:port)
-          profile.proxy = Selenium::WebDriver::Proxy.new(
-            http: "#{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)}",
-            ssl:  "#{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)}"
+          proxy_config                  =     Selenium::WebDriver::Proxy.new(
+            http:   "#{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)}",
+            ssl:    "#{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)}",
+            socks:  "#{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)}"
           )
+          
+          proxy_config.socks_username   =   self.proxy.fetch(:username) unless self.proxy.fetch(:username, nil).to_s.empty?
+          proxy_config.socks_password   =   self.proxy.fetch(:password) unless self.proxy.fetch(:password, nil).to_s.empty?
+          
+          profile.proxy                 =   proxy_config
         
           log("Will use proxy #{self.proxy.fetch(:host)}:#{self.proxy.fetch(:port)} to initiate the request.")
         end

data/lib/browsed/version.rb

@@ -1,3 +1,3 @@
 module Browsed
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: browsed
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Sebastian
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-27 00:00:00.000000000 Z
+date: 2019-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -168,6 +168,7 @@ files:
 - lib/browsed/configuration.rb
 - lib/browsed/constants.rb
 - lib/browsed/errors.rb
+- lib/browsed/extensions/chrome/proxy_authentication.rb
 - lib/browsed/firefox.rb
 - lib/browsed/manager.rb
 - lib/browsed/poltergeist.rb