crabfarm 0.2.5 → 0.3.0

data/lib/crabfarm/driver_pool.rb

@@ -0,0 +1,32 @@
+module Crabfarm
+  class DriverPool
+
+    def initialize(_factory)
+      @factory = _factory
+      @drivers = Hash.new
+    end
+
+    def driver(_session_id=nil)
+      _session_id ||= :default_driver
+      driver = @drivers[_session_id.to_sym]
+      driver = @drivers[_session_id.to_sym] = @factory.build_driver(_session_id) if driver.nil?
+      driver
+    end
+
+    def reset(_session_id=nil)
+      if _session_id.nil?
+        @drivers.values.each { |d| @factory.release_driver d }
+        @drivers = Hash.new
+      else
+        _session_id = _session_id.to_sym
+        driver = @drivers.delete _session_id
+        @factory.release_driver driver unless driver.nil?
+      end
+    end
+
+    def release
+      reset
+    end
+
+  end
+end

data/lib/crabfarm/dsl/surfer/surf_context.rb

@@ -3,12 +3,13 @@ module Crabfarm
     module Surfer
       class SurfContext < SearchContext
 
-        def_delegators :@bucket, :setup
+        attr_reader :driver
+
         def_delegators 'driver.navigate', :back, :forward, :refresh
 
-        def initialize(_bucket)
+        def initialize(_driver)
           super nil, self
-          @bucket = _bucket
+          @driver = _driver
         end
 
         def root
@@ -23,14 +24,6 @@ module Crabfarm
           driver.page_source
         end
 
-        def driver
-          @bucket.original
-        end
-
-        def quit
-          @bucket.reset
-        end
-
         def current_uri
           URI.parse driver.current_url
         end
@@ -41,20 +34,7 @@ module Crabfarm
 
         def goto(_url, _params=nil)
           _url += "?#{_params.to_query}" if _params
-          retries = 0
-
-          loop do
-            begin
-              @bucket.reset if retries > 0
-              driver.get(_url)
-              break
-            rescue Timeout::Error #, Selenium::WebDriver::Error::UnknownError
-              # TODO: log this
-              raise if retries >= max_retries
-              retries += 1
-              sleep 1.0
-            end
-          end
+          driver.get(_url)
         end
       end
     end

data/lib/crabfarm/engines/async_state_manager.rb

@@ -108,7 +108,7 @@ module Crabfarm
                 logger.info "Transitioning state: #{@next_state_name}"
                 @elapsed = Benchmark.measure do
                   ActiveSupport::Dependencies.clear
-                  @doc = TransitionService.apply_state(@context, @next_state_name, @next_state_params).output_as_json
+                  @doc = TransitionService.transition(@context, @next_state_name, @next_state_params).document
                 end.real
 
                 logger.info "Transitioned in #{@elapsed.real}"

data/lib/crabfarm/engines/sync_state_manager.rb

@@ -28,7 +28,7 @@ module Crabfarm
           output = { name: _name, params: _params }
 
           output[:elapsed] = Benchmark.measure do
-            output[:doc] = TransitionService.apply_state(@context, _name, _params).output_as_json
+            output[:doc] = TransitionService.transition(@context, _name, _params).document
           end
 
           OpenStruct.new output

data/lib/crabfarm/forked_navigator.rb

@@ -0,0 +1,31 @@
+module Crabfarm
+  class ForkedNavigator < Delegator
+
+    def initialize(_context, _parent, _browser_name, _mutex)
+      @context = _context
+      @parent = _parent
+      @browser_name = _browser_name
+      @mutex = _mutex
+
+      super @parent
+    end
+
+    def browser
+      @browser ||= @context.pool.driver(@browser_name)
+    end
+
+    def synchronize
+      @mutex.synchronize {
+        yield
+      }
+    end
+
+    def __getobj__
+      @parent
+    end
+
+    def __setobj__(obj)
+      @parent = obj
+    end
+  end
+end

data/lib/crabfarm/modes/console.rb

@@ -20,14 +20,14 @@ module Crabfarm
           super
         end
 
-        def transition(_name=nil, _params={})
+        def navigate(_name=nil, _params={})
           if _name.nil?
-            puts "Must provide a state name".color(:red)
+            puts "Must provide a navigator name".color(:red)
             return
           end
 
           begin
-            puts "Transitioning to #{_name.to_s.camelize} state"
+            puts "Navigating to #{_name.to_s.camelize} state"
             output = super
 
             puts "State changed, generated document:"
@@ -44,7 +44,7 @@ module Crabfarm
           puts "Ejem..."
         end
 
-        alias :t :transition
+        alias :n :navigate
         alias :r :reset
       end
 

data/lib/crabfarm/modes/generator.rb

@@ -23,8 +23,9 @@ module Crabfarm
           path(_name, '.crabfarm').render('dot_crabfarm', binding)
           path(_name, 'boot.rb').render('boot.rb', binding)
           path(_name, 'bin', 'crabfarm').render('crabfarm_bin', binding, 0755)
-          path(_name, 'app', 'parsers', '.gitkeep').render('dot_gitkeep')
-          path(_name, 'app', 'states', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'app', 'navigators', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'app', 'reducers', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'app', 'structs', '.gitkeep').render('dot_gitkeep')
           path(_name, 'app', 'helpers', '.gitkeep').render('dot_gitkeep')
           path(_name, 'spec', 'spec_helper.rb').render('spec_helper.rb', binding)
           path(_name, 'spec', 'snapshots', '.gitkeep').render('dot_gitkeep')
@@ -34,26 +35,38 @@ module Crabfarm
         end
       end
 
-      def generate_state(_target, _class_name)
+      def generate_navigator(_target, _class_name, _skip_reducer=false)
         validate_class_name _class_name
 
         route = Utils::Naming.route_from_constant _class_name
         with_base_path _target do
-          binding = { state_class: _class_name }
-          path(*(['app', 'states'] + route[0...-1] + [route.last + '.rb'])).render('state.rb', binding)
-          path(*(['spec', 'states'] + route[0...-1] + [route.last + '_spec.rb'])).render('state_spec.rb', binding)
+          binding = { navigator_class: _class_name }
+          path(*(['app', 'navigators'] + route[0...-1] + [route.last + '.rb'])).render('navigator.rb', binding)
+          path(*(['spec', 'navigators'] + route[0...-1] + [route.last + '_spec.rb'])).render('navigator_spec.rb', binding)
+        end
+
+        generate_reducer(_target, _class_name) unless _skip_reducer
+      end
+
+      def generate_reducer(_target, _class_name)
+        validate_class_name _class_name
+
+        _class_name = _class_name + 'Reducer'
+        route = Utils::Naming.route_from_constant _class_name
+        with_base_path _target do
+          binding = { reducer_class: _class_name }
+          path(*(['app', 'reducers'] + route[0...-1] + [route.last + '.rb'])).render('reducer.rb', binding)
+          path(*(['spec', 'reducers'] + route[0...-1] + [route.last + '_spec.rb'])).render('reducer_spec.rb', binding)
         end
       end
 
-      def generate_parser(_target, _class_name)
+      def generate_struct(_target, _class_name)
         validate_class_name _class_name
 
-        _class_name = _class_name + 'Parser'
         route = Utils::Naming.route_from_constant _class_name
         with_base_path _target do
-          binding = { parser_class: _class_name }
-          path(*(['app', 'parsers'] + route[0...-1] + [route.last + '.rb'])).render('parser.rb', binding)
-          path(*(['spec', 'parsers'] + route[0...-1] + [route.last + '_spec.rb'])).render('parser_spec.rb', binding)
+          binding = { struct_class: _class_name }
+          path(*(['app', 'structs'] + route[0...-1] + [route.last + '.rb'])).render('struct.rb', binding)
         end
       end
 

data/lib/crabfarm/rspec.rb

@@ -3,9 +3,9 @@ module Crabfarm
 
     class Error < Crabfarm::Error; end
 
-    def parse(_snapshot=nil, _options={})
+    def reduce(_snapshot=nil, _options={})
 
-      raise Error.new "Crawl is only available in parser specs" unless described_class < Crabfarm::BaseParser
+      raise Error.new "'reduce' is only available in reducer specs" unless described_class < Crabfarm::BaseReducer
 
       if _snapshot.is_a? Hash
         raise ArgumentException.new 'Invalid arguments' unless _options.nil?
@@ -17,42 +17,44 @@ module Crabfarm
       raise Error.new "Snapshot does not exist #{_snapshot}" unless File.exist? snapshot_path
 
       data = File.read snapshot_path
-      parser = described_class.new data, _options
-      parser.parse
-      parser
+      reducer = described_class.new data, _options
+      reducer.run
+      reducer
     end
 
-    def crawl(_state=nil, _params={})
+    def navigate(_name=nil, _params={})
 
-      raise Error.new "Crawl is only available in state specs" if @context.nil?
+      raise Error.new "'navigate' is only available in navigator specs" if @context.nil?
 
-      if _state.is_a? Hash
-        _params = _state
-        _state = nil
+      if _name.is_a? Hash
+        _params = _name
+        _name = nil
       end
 
-      if _state.nil?
-        return nil unless described_class < BaseState # TODO: maybe raise an error here.
-        @state = @last_state = TransitionService.apply_state @context, described_class, _params
+      if _name.nil?
+        return nil unless described_class < BaseNavigator # TODO: maybe raise an error here.
+        @state = @last_state = TransitionService.transition @context, described_class, _params
       else
-        @last_state = TransitionService.apply_state @context, _state, _params
+        @last_state = TransitionService.transition @context, _name, _params
       end
     end
 
     def state
-      @state ||= crawl
+      @state ||= navigate
     end
 
+    alias :navigator :state
+
     def last_state
       @last_state
     end
 
-    def parser
-      @parser ||= parse
+    def reducer
+      @reducer ||= reduce
     end
 
-    def driver(_session_id=nil)
-      @context.pool.driver(_session_id)
+    def browser(_session_id=nil)
+      @context.pool.browser(_session_id)
     end
 
   end
@@ -62,13 +64,13 @@ RSpec.configure do |config|
   config.include Crabfarm::RSpec
 
   config.around(:example) do |example|
-    if described_class < Crabfarm::BaseParser
-      if example.metadata[:parsing] || example[:parsing_with_params]
-        @parser = parse example.metadata[:parsing], example.metadata[:parsing_with_params] || {}
+    if described_class < Crabfarm::BaseReducer
+      if example.metadata[:reducing] || example[:reducing_with_params]
+        @reducer = reduce example.metadata[:reducing], example.metadata[:reducing_with_params] || {}
       end
       example.run
-    elsif described_class < Crabfarm::BaseState
-      Crabfarm::ContextFactory.with_context example.metadata[:crawling] do |ctx|
+    elsif described_class < Crabfarm::BaseNavigator
+      Crabfarm::ContextFactory.with_context example.metadata[:navigating] do |ctx|
         @context = ctx
         example.run
       end

data/lib/crabfarm/strategies.rb

@@ -2,28 +2,34 @@ module Crabfarm
   module Strategies
 
     class Loader
-      def initialize(_name, _klass, _pkg, _deps)
+      def initialize(_name, _klass, _options={})
         @name = _name
         @klass = _klass
-        @pkg = _pkg
-        @deps = _deps
+
+        @pkg = if _options.key? :require
+          _options[:require]
+        elsif @klass.is_a? String
+          Utils::Naming.route_from_constant(@klass).join('/')
+        else nil end
+
+        @requirements = Array(_options[:dependencies]) if _options.key? :dependencies
       end
 
       def load
-        load_dependencies
+        load_requirements unless @requirements.nil?
         require @pkg if @pkg
         if @klass.is_a? String then Object.const_get @klass else @klass end
       end
 
     private
 
-      def load_dependencies
-        @deps.each do |dep|
+      def load_requirements
+        @requirements.each do |dep|
           begin
             require dep
             # TODO: check dependency version!
           rescue LoadError
-            raise ConfigurationError.new "Missing #{@name} dependency, please add `gem \"#{dep}\"` to the crawler's Gemfile"
+            raise ConfigurationError.new "Could not find #{@name} dependency, maybe you forgot to add `gem \"#{dep}\"` to the crawler's Gemfile?"
           end
         end
       end
@@ -31,9 +37,9 @@ module Crabfarm
 
     @@register = {}
 
-    def self.register(_cat, _name, _klass, _pkg=nil, _deps=[])
+    def self.register(_cat, _name, _klass, _options={})
       full_name = _cat.to_s + ':' + _name.to_s
-      @@register[full_name] = Loader.new(full_name, _klass, _pkg, _deps)
+      @@register[full_name] = Loader.new(full_name, _klass, _options)
     end
 
     def self.load(_cat, _name)

data/lib/crabfarm/templates/Crabfile.erb

@@ -1,44 +1,39 @@
+# The selected browser engine for navigators to be used throughout the crawler
+# Other options (may require instalation of additional libraries): :phantomjs, :remote, :chrome, :firefox
+set_browser :phantomjs
 
-# The default crawling dsl to use in states and parsers, can be overriden in each component using the `browser_dsl :dsl` modifier
-# Available options are :surfer, :watir and :capybara. Both watir and capybara require an additional gem to be added to Gemfile
-set_browser_dsl :surfer
-
-# The default parser engine for parsers that do not specify one.
+# The default parser engine for reducers that do not specify one.
 # Available options are :nokogiri and :pdf_parser. :pdf_parser requires an additional gem to be added to Gemfile
-# set_parser_engine :nokogiri
-
-# Change the defaut output builder used in a state to generate the output document.
-# Available options are :hash, :ostruct, :jbuilder. :jbuilder requires an additional gem to be added to Gemfile
-# set_output_builder :hash
+set_parser :nokogiri
 
 # The path where every crawler log is stored.
 set_log_path 'logs'
 
+# Set crawler proxy, this setting is overrided when running the crawler in crabfarm.io
+# set_proxy 'the.proxy.address'
 
-# General driver configuration
+# General webdriver configuration
 ########################################
 
-# The selected selenium driver to be using throughout the crawler
-# Other options (may require instalation of additional libraries): :phantomjs, :remote, :chrome, :firefox
-set_driver :phantomjs
+# The following parameters only apply if using a webdriver based driver
+
+# Selects the webdriver wrapper library to be used, options are :surfer, :watir and :capybara.
+# Both watir and capybara require an additional gem to be added to Gemfile
+set_webdriver_dsl :surfer
 
 # Set the selected webdriver capabilities (check the driver documentation for more details)
-# set_driver_capabilities
+# set_webdriver_capabilities
 
 # Set the browser window width
-# set_driver_window_width 1280
+# set_webdriver_window_width 1280
 
 # Set the browser window height
-# set_driver_window_height 800
-
-# Set the driver proxy address
-# set_proxy
-
+# set_webdriver_window_height 800
 
 # Phantom launcher configuration
 ########################################
 
-# The following parameters only apply if using the :phantomjs driver
+# The following parameters only apply if using the :phantomjs webdriver
 
 # Make phantom load images when requesting documents
 # set_phantom_load_images false
@@ -53,16 +48,16 @@ set_driver :phantomjs
 # Remote driver options
 ########################################
 
-# The following parameters only apply if using the :remote driver
+# The following parameters only apply if using the :remote webdriver
 
 # Remote driver host
-# set_driver_host www.myseleniumgrid.com
+# set_webdriver_host www.myseleniumgrid.com
 
 # Remote driver port
-# set_driver_port 8080
+# set_webdriver_port 8080
 
 # Remote driver response timeout, in seconds
-# set_driver_remote_timeout 120
+# set_webdriver_remote_timeout 120
 
 
 # Recording configuration