crabfarm 0.7.9 → 0.7.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a3f375503cc8b1d8107bfdb7df79ec9ce923852
-  data.tar.gz: 7a0d201d17f5983f37426285e88f72f4538dee70
+  metadata.gz: 61b08ac4fbd03cce9915e677accfd155a32df06d
+  data.tar.gz: c6dfcfde7084e9a169c3b841b0db323f91e618f0
 SHA512:
-  metadata.gz: a6b929e335feddffdeef3438a216978d0ae0b887b58d56f4c4cb031ddd70d680af26c04e2f08731b7a4b4372450cefd753ef29f32a57da7bea3f892931342600
-  data.tar.gz: 758d3b2c5f11ed4f94cd4fa2b5d587c71d21035a0ed87870cafcd89d48e50ed2443e770c7386bd8d5140d4e4f014f46a51af1585c3ed2ff10e03ebaae7eb4e7e
+  metadata.gz: 1a6e29253f64e0c783b7ad257cbcd66f247b50310f275b3ac1384ced3c5fb8fba1942dfc64da72833c40c414e2251405d39f01ee40d1b1de2f0cd10dad873b86
+  data.tar.gz: 84ac0447cfdacf3bb6e04ea73b937a157a13954ba5a5325a10b60d61d8261b9a46315acdd38fd522daa7e592dc6375cfedece54fb6779ae2569f41b26b6198cc

data/lib/crabfarm/adapters/browser/abstract_webdriver.rb

@@ -7,9 +7,10 @@ module Crabfarm
 
         attr_accessor :config
 
-        def initialize(_proxy=nil)
+        def initialize(_proxy = nil, _proxy_auth = nil)
           @config = load_driver_config
           @config[:proxy] = _proxy
+          @config[:proxy_auth] = _proxy_auth
         end
 
         def build_driver(_session_id)

data/lib/crabfarm/adapters/browser/chenso.rb

@@ -4,10 +4,10 @@ module Crabfarm
   module Adapters
     module Browser
       class Chenso < Base
-
-        def initialize(_proxy=nil)
+        def initialize(_proxy = nil, _proxy_auth = nil)
           @config = load_chenso_config
           @config[:proxy] = _proxy
+          @config[:proxy_auth] = _proxy_auth
         end
 
         def build_driver(_session_id)
@@ -22,10 +22,9 @@ module Crabfarm
 
         def load_chenso_config
           {
-            # nothing for now
+            user_agent: Crabfarm.config.user_agent
           }
         end
-
       end
     end
   end

data/lib/crabfarm/adapters/browser/noop.rb

@@ -4,14 +4,12 @@ module Crabfarm
   module Adapters
     module Browser
       class Noop < Base
-
-        def initialize(_proxy=nil)
+        def initialize(_proxy = nil, _proxy_user = nil)
         end
 
         def build_driver(_session_id)
           _session_id || :noop
         end
-
       end
     end
   end

data/lib/crabfarm/configuration.rb

@@ -10,6 +10,8 @@ module Crabfarm
       [:parser, :string, 'Default parser engine used by reducers'],
       [:log_path, :string, 'Path where logs should be stored'],
       [:proxy, :string, 'If given, a proxy is used to connect to the internet if driver supports it'],
+      [:proxy_auth, :string, 'Proxy authentication parameters as user:password'],
+      [:user_agent, :string, 'Allows overriding default driver user agent, only available in chenso'],
 
       # Webdriver configuration parameters
       [:webdriver_host, :string, 'Remote host, only available in driver: remote'],
@@ -56,6 +58,8 @@ module Crabfarm
         driver_factory: nil,
         log_path: nil,
         proxy: nil,
+        proxy_auth: nil,
+        user_agent: nil,
         webdriver_capabilities: nil,
         webdriver_host: 'localhost',
         webdriver_port: '8080',
@@ -87,8 +91,7 @@ module Crabfarm
     def crabtrap_config
       {
         bin_path: crabtrap_bin_path,
-        log_level: crabtrap_log_level,
-        proxy: proxy
+        log_level: crabtrap_log_level
       }
     end
 

data/lib/crabfarm/context.rb

@@ -48,7 +48,7 @@ module Crabfarm
 
     def init_browser_adapter
       if @browser_adapter.nil?
-        @browser_adapter = build_browser_adapter proxy
+        @browser_adapter = build_browser_adapter proxy, proxy_auth
         @browser_adapter.prepare_driver_services
       end
     end
@@ -67,14 +67,18 @@ module Crabfarm
       @pool = nil
     end
 
-    def build_browser_adapter(_proxy)
-      Strategies.load(:browser, config.browser).new _proxy
+    def build_browser_adapter(_proxy, _proxy_auth)
+      Strategies.load(:browser, config.browser).new(_proxy, _proxy_auth)
     end
 
     def proxy
       config.proxy
     end
 
+    def proxy_auth
+      config.proxy_auth
+    end
+
     def config
       Crabfarm.config
     end

data/lib/crabfarm/crabtrap_context.rb

@@ -2,7 +2,6 @@ require 'crabfarm/crabtrap_runner'
 
 module Crabfarm
   class CrabtrapContext < Context
-
     attr_accessor :mode
 
     def initialize(_mode=:pass, _path=nil)
@@ -78,13 +77,15 @@ module Crabfarm
     end
 
     def proxy
-      # just step over configuration proxy
       proxy_address
     end
 
+    def proxy_auth
+      nil
+    end
+
     def proxy_address
       "127.0.0.1:#{@port}"
     end
-
   end
 end

data/lib/crabfarm/live/context.rb

@@ -15,9 +15,13 @@ module Crabfarm
         "127.0.0.1:#{@manager.proxy_port}"
       end
 
+      def proxy_auth
+        nil
+      end
+
     private
 
-      def build_browser_adapter(_proxy)
+      def build_browser_adapter(_proxy, _proxy_auth)
         # use a special browser adapter to override primary driver
         return BrowserAdapter.new @manager
       end

data/lib/crabfarm/modes/generator.rb

@@ -32,6 +32,7 @@ module Crabfarm
           path(_name, 'spec', 'mementos', '.gitkeep').render('dot_gitkeep')
           path(_name, 'spec', 'integration', '.gitkeep').render('dot_gitkeep')
           path(_name, 'logs', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'README.md').render('README.md', binding)
         end
       end
 

data/lib/crabfarm/support/webdriver_factory.rb

@@ -1,61 +1,50 @@
 require 'selenium-webdriver'
 
 module Crabfarm
- module Support
-  module WebdriverFactory
-    extend self
+  module Support
+    module WebdriverFactory
+      extend self
 
-    def build_chrome_driver(_options={})
-      capabilities = Selenium::WebDriver::Remote::Capabilities.chrome
+      def build_chrome_driver(_options={})
+        capabilities = Selenium::WebDriver::Remote::Capabilities.chrome
+        capabilities.proxy = build_proxy(_options) if _options[:proxy].present?
 
-      if _options[:proxy].present?
-        capabilities.proxy = Selenium::WebDriver::Proxy.new({
-          :http => _options[:proxy],
-          :ssl => _options[:proxy]
-        })
+        setup_webdriver Selenium::WebDriver.for(:chrome, detach: false, desired_capabilities: capabilities), _options
       end
 
-      common_setup Selenium::WebDriver.for(:chrome, detach: false, desired_capabilities: capabilities), _options
-    end
-
-    def build_firefox_driver(_options={})
-      capabilities = Selenium::WebDriver::Remote::Capabilities.firefox
+      def build_firefox_driver(_options={})
+        capabilities = Selenium::WebDriver::Remote::Capabilities.firefox
+        capabilities.proxy = build_proxy(_options) if _options[:proxy].present?
 
-      if _options[:proxy].present?
-        capabilities.proxy = Selenium::WebDriver::Proxy.new({
-          :http => _options[:proxy],
-          :ssl => _options[:proxy]
-        })
+        setup_webdriver Selenium::WebDriver.for(:firefox, desired_capabilities: capabilities), _options
       end
 
-      common_setup Selenium::WebDriver.for(:firefox, desired_capabilities: capabilities), _options
-    end
+      def build_remote_driver(_options={})
+        client = Selenium::WebDriver::Remote::Http::Default.new
+        client.timeout = _options[:remote_timeout]
+        client.proxy = build_proxy(_options) if _options[:proxy].present?
+
+        setup_webdriver(Selenium::WebDriver.for(:remote, {
+          :url => _options[:remote_host],
+          :http_client => client,
+          :desired_capabilities => _options[:capabilities] || Selenium::WebDriver::Remote::Capabilities.firefox
+        }), _options)
+      end
 
-    def build_remote_driver(_options={})
-      client = Selenium::WebDriver::Remote::Http::Default.new
-      client.timeout = _options[:remote_timeout]
+    private
 
-      if _options[:proxy].present?
-        client.proxy = Selenium::WebDriver::Proxy.new({
+      def build_proxy(_options)
+        # TODO: support authentication
+        Selenium::WebDriver::Proxy.new({
           :http => _options[:proxy],
           :ssl => _options[:proxy]
         })
       end
 
-      common_setup(Selenium::WebDriver.for(:remote, {
-        :url => _options[:remote_host],
-        :http_client => client,
-        :desired_capabilities => _options[:capabilities] || Selenium::WebDriver::Remote::Capabilities.firefox
-      }), _options)
-    end
-
-  private
-
-    def common_setup(_driver, _options)
-      _driver.manage.window.resize_to(_options[:window_width], _options[:window_height]) rescue nil
-      return _driver
+      def setup_webdriver(_driver, _options)
+        _driver.manage.window.resize_to(_options[:window_width], _options[:window_height]) rescue nil
+        return _driver
+      end
     end
-
   end
- end
 end

data/lib/crabfarm/templates/Crabfile.erb

@@ -12,6 +12,12 @@ set_log_path 'logs'
 # Set crawler proxy, this setting is overrided when running the crawler in crabfarm.io
 # set_proxy 'the.proxy.address'
 
+# Set crawler proxy authentication, this setting is overrided when running the crawler in crabfarm.io
+# set_proxy_auth 'user:password'
+
+# Set the crawler's user agent string
+# set_user_agent 'MyCrawler'
+
 # General webdriver configuration
 ########################################
 

data/lib/crabfarm/templates/Gemfile.erb

@@ -1,7 +1,7 @@
 source 'https://rubygems.org'
 
 gem "crabfarm", '<%= version %>'
-gem "pincers", '~> 0.7'
+gem "pincers", '~> 0.7', ">= 0.7.11"
 
 # Comment this if not using a nokogiri based parser or browser
 gem 'nokogiri', "~> 1.6"

data/lib/crabfarm/templates/README.md.erb

@@ -0,0 +1,44 @@
+![Crabfarm](http://crabfarm.io/img/teaser-bg-3.png)
+
+## <%= name %>
+
+This is a crawler created using the [Crabfarm framework](http://crabfarm.io/#/teaser). It is composed of navigators which allow you to access different sections of a website to extract infomation or perform actions.
+
+To learn more about how this crawler was put together read the [documentation](http://github.com/platanus/crabfarm-gem).
+
+### Deploy
+
+To deploy <%= name %> first you must have an active account for the [Crabfarm Grid](https://grid.crabfarm.io).
+
+If you already have an account, simply do:
+```shell
+crabfarm p
+```
+
+This will ask you for you credentials if it haven't and upload it to the grid.
+
+### Use
+
+To use the crawler from Ruby you can install cangrejo-gem:
+
+```
+gem install 'cangrejo'
+```
+
+and then
+
+```ruby
+require 'cangrejo'
+Cangrejo.connect 'org/repo' do |session|
+  session.navigate(:navigator_name, parameter_name: 'hello')
+end
+```
+
+For more information, visit the [cangrejo-gem repository](https://github.com/platanus/cangrejo-gem).
+
+If you prefer NodeJS instead, there is [Camaron](https://github.com/platanus/camaron)
+
+
+### Navigators
+
+Take a look at the [usage examples](tree/master/spec/navigators)

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.7.9"
+  VERSION = "0.7.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.7.9
+  version: 0.7.11
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-05 00:00:00.000000000 Z
+date: 2016-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -223,6 +223,9 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.7'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.11
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -230,6 +233,9 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.7'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.11
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -535,6 +541,7 @@ files:
 - lib/crabfarm/support/webdriver_factory.rb
 - lib/crabfarm/templates/Crabfile.erb
 - lib/crabfarm/templates/Gemfile.erb
+- lib/crabfarm/templates/README.md.erb
 - lib/crabfarm/templates/boot.rb.erb
 - lib/crabfarm/templates/crabfarm_bin.erb
 - lib/crabfarm/templates/dot_crabfarm.erb
@@ -576,7 +583,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.1
 signing_key: 
 specification_version: 4
 summary: Crabfarm is a TDD oriented web scrapping framework