crabfarm 0.2.5 → 0.3.0

data/lib/crabfarm/base_navigator.rb

@@ -0,0 +1,78 @@
+require 'thwait'
+require 'crabfarm/forked_navigator'
+require "crabfarm/assertion/context"
+
+module Crabfarm
+  class BaseNavigator
+    include Assertion::Context
+    extend Forwardable
+
+    attr_reader :params
+
+    def_delegators '@context', :http
+    def_delegators '@context.store', :get, :fetch
+
+    def initialize(_context, _params)
+      @context = _context
+      @params = _params
+    end
+
+    def browser(_name=nil)
+      @context.pool.driver(_name)
+    end
+
+    def download(_url)
+      @context.http.get(_url).body
+    end
+
+    def run
+      raise NotImplementedError.new
+    end
+
+    def reduce(_target=nil, _options={})
+      if _target.is_a? Hash
+        _options = _target
+        _target = browser
+      elsif _target.nil?
+        _target = browser
+      end
+
+      reduce_using(_options.delete(:using) || self.class.name, _target, _options)
+    end
+
+    alias :reduce_with_defaults :reduce
+
+    def fork_each(_enumerator, &_block)
+      session_id = 0
+      mutex = Mutex.new
+      ths = _enumerator.map do |value|
+        session_id += 1
+        start_forked_navigation("th_session_#{session_id}", value, _block, mutex)
+      end
+      ThreadsWait.all_waits(*ths)
+    end
+
+  private
+
+    def reduce_using(_reducer_class, _target, _options={})
+      if _reducer_class.is_a? String or _reducer_class.is_a? Symbol
+        _reducer_class = (Utils::Naming.decode_crabfarm_uri(_reducer_class.to_s) + 'Reducer').constantize
+      end
+
+      reducer = _reducer_class.new _target, @params.merge(_options)
+      reducer.run
+      reducer
+    end
+
+    def start_forked_navigation(_name, _value, _block, _mutex)
+      Thread.new {
+        fork = ForkedNavigator.new @context, self, _name, _mutex
+        begin
+          fork.instance_exec _value, &_block
+        ensure
+          @context.pool.reset _name
+        end
+      }
+    end
+  end
+end

data/lib/crabfarm/base_reducer.rb

@@ -0,0 +1,68 @@
+require "crabfarm/assertion/fields"
+
+module Crabfarm
+  class BaseReducer < Delegator
+    include Assertion::Fields
+
+    attr_reader :params, :document
+
+    def self.use_parser(_parser_name)
+      @parser_name = _parser_name
+    end
+
+    def self.parser
+      @parser ||= Strategies.load(:parser, @parser_name || Crabfarm.config.parser)
+    end
+
+    def self.snapshot_path(_name=nil)
+      _name = self.to_s.underscore if _name.nil?
+      File.join(GlobalState.snapshots_path, _name + '.' + parser.format)
+    end
+
+    def parser
+      self.class.parser
+    end
+
+    def initialize(_target, _params)
+      reset_fields
+
+      @parsed_data = parser.preprocess_parsing_target _target
+      @document = parser.parse @parsed_data
+      @params = _params
+
+      super @document
+    end
+
+    def run
+      raise NotImplementedError.new
+    end
+
+    def take_snapshot(_name=nil)
+      file_path = self.class.snapshot_path _name
+
+      dir_path = file_path.split(File::SEPARATOR)[0...-1]
+      FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
+
+      File.write file_path, @parsed_data
+      file_path
+    end
+
+    def take_snapshot_and_fail(_name=nil)
+      file_path = take_snapshot _name
+      raise ArgumentError.new "New snapshot for #{self.class.to_s} generated in '#{file_path}'"
+    end
+
+    def as_json(_options=nil)
+      field_hash
+    end
+
+    def __getobj__
+      @document
+    end
+
+    def __setobj__(obj)
+      @document = obj
+    end
+
+  end
+end

data/lib/crabfarm/base_struct.rb

@@ -0,0 +1,17 @@
+require "crabfarm/assertion/fields"
+
+module Crabfarm
+  class BaseStruct
+    include Assertion::Fields
+
+    def initialize(_values={})
+      reset_fields
+      _values.each { |k,v| send("#{k}=", v) }
+    end
+
+    def as_json(_options=nil)
+      field_hash
+    end
+
+  end
+end

data/lib/crabfarm/cli.rb

@@ -85,23 +85,33 @@ module Crabfarm
         end
       end
 
-      c.desc "Generates a new crabfarm parser and parser spec"
-      c.command :parser do |parser|
-        parser.action do |global_options,options,args|
+      c.desc "Generates a new crabfarm navigator and navigator spec"
+      c.command :navigator do |sub|
+        sub.action do |global_options,options,args|
           next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
 
           require "crabfarm/modes/generator"
-          Crabfarm::Modes::Generator.generate_parser(GlobalState.app_path, args[0])
+          Crabfarm::Modes::Generator.generate_navigator(GlobalState.app_path, args[0])
         end
       end
 
-      c.desc "Generates a new crabfarm state and parser spec"
-      c.command :state do |parser|
-        parser.action do |global_options,options,args|
+      c.desc "Generates a new crabfarm reducer and reducer spec"
+      c.command :reducer do |sub|
+        sub.action do |global_options,options,args|
           next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
 
           require "crabfarm/modes/generator"
-          Crabfarm::Modes::Generator.generate_state(GlobalState.app_path, args[0])
+          Crabfarm::Modes::Generator.generate_reducer(GlobalState.app_path, args[0])
+        end
+      end
+
+      c.desc "Generates a new crabfarm struct"
+      c.command :reducer do |sub|
+        sub.action do |global_options,options,args|
+          next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
+
+          require "crabfarm/modes/generator"
+          Crabfarm::Modes::Generator.generate_struct(GlobalState.app_path, args[0])
         end
       end
     end

data/lib/crabfarm/configuration.rb

@@ -5,21 +5,20 @@ module Crabfarm
     class Option < Struct.new(:name, :type, :text); end
 
     OPTIONS = [
-      [:browser_dsl, :string, 'Default browser dsl used by states'],
-      [:parser_engine, :string, 'Default parser engine used by parsers'],
-      [:output_builder, :string, 'Default json output builder used by states'],
-      [:driver_factory, :mixed, 'Driver factory, disabled if phantom_mode is used'],
+      # Global options
+      [:browser, ['chrome', 'firefox', 'phantomjs', 'remote'], 'Browser engine to be used by navigators, common options: phantomjs, chrome, firefox, remote.'],
+      [:parser, :string, 'Default parser engine used by reducers'],
       [:log_path, :string, 'Path where logs should be stored'],
       [:proxy, :string, 'If given, a proxy is used to connect to the internet if driver supports it'],
 
-      # Default driver configuration parameters
-      [:driver, ['chrome', 'firefox', 'phantomjs', 'remote'], 'Webdriver to be user, common options: chrome, firefox, phantomjs, remote.'],
-      [:driver_host, :string, 'Remote host, only available in driver: remote'],
-      [:driver_port, :integer, 'Remote port, only available in driver: remote'],
-      [:driver_capabilities, :mixed, 'Driver capabilities, depends on selected driver.'],
-      [:driver_remote_timeout, :float, 'Request timeout in seconds, only available for remote or phatomjs driver.'],
-      [:driver_window_width, :integer, 'Initial browser window width.'],
-      [:driver_window_height, :integer, 'Initial browser window height.'],
+      # Webdriver configuration parameters
+      [:webdriver_host, :string, 'Remote host, only available in driver: remote'],
+      [:webdriver_port, :integer, 'Remote port, only available in driver: remote'],
+      [:webdriver_capabilities, :mixed, 'Driver capabilities, depends on selected driver.'],
+      [:webdriver_remote_timeout, :float, 'Request timeout in seconds, only available for remote or phatomjs driver.'],
+      [:webdriver_window_width, :integer, 'Initial browser window width.'],
+      [:webdriver_window_height, :integer, 'Initial browser window height.'],
+      [:webdriver_dsl, :string, 'Webdriver wrapper to use, built in options are watir and surfer'],
 
       # Phantom launcher configuration
       [:phantom_load_images, :boolean, 'Phantomjs image loading, only for phantomjs driver.'],
@@ -50,19 +49,18 @@ module Crabfarm
 
     def reset
       @values = {
-        browser_dsl: :surfer,
-        parser_engine: :nokogiri,
-        output_builder: :hash,
+        browser: 'phantomjs',
+        parser: :nokogiri,
         driver_factory: nil,
         log_path: nil,
         proxy: nil,
-        driver: 'phantomjs',
-        driver_capabilities: Selenium::WebDriver::Remote::Capabilities.firefox,
-        driver_host: 'localhost',
-        driver_port: '8080',
-        driver_remote_timeout: 120,
-        driver_window_width: 1280,
-        driver_window_height: 800,
+        webdriver_capabilities: nil,
+        webdriver_host: 'localhost',
+        webdriver_port: '8080',
+        webdriver_remote_timeout: 120,
+        webdriver_window_width: 1280,
+        webdriver_window_height: 800,
+        webdriver_dsl: :surfer,
         phantom_load_images: false,
         phantom_ssl: 'any',
         phantom_bin_path: 'phantomjs',
@@ -75,38 +73,13 @@ module Crabfarm
       @values.merge! _options
     end
 
-    def driver_remote_host
-      if driver_host then nil
-      elsif driver_port then "http://#{driver_host}"
-      else "http://#{driver_host}:#{driver_port}"
+    def webdriver_remote_host
+      if webdriver_host then nil
+      elsif webdriver_port then "http://#{webdriver_host}"
+      else "http://#{webdriver_host}:#{webdriver_port}"
       end
     end
 
-    def driver_config
-      {
-        name: driver,
-        proxy: proxy,
-        capabilities: driver_capabilities,
-        remote_host: driver_remote_host,
-        remote_timeout: driver_remote_timeout,
-        window_width: driver_window_width,
-        window_height: driver_window_height
-      }
-    end
-
-    def phantom_mode_enabled?
-      driver.to_s == 'phantomjs'
-    end
-
-    def phantom_config
-      {
-        load_images: phantom_load_images,
-        proxy: proxy,
-        ssl: phantom_ssl,
-        bin_path: phantom_bin_path
-      }
-    end
-
     def crabtrap_config
       {
         bin_path: crabtrap_bin_path,

data/lib/crabfarm/context.rb

@@ -32,7 +32,7 @@ module Crabfarm
   private
 
     def load_services
-      init_phantom_if_required
+      init_driver_factory
       init_driver_pool
       init_http_client
     end
@@ -43,71 +43,47 @@ module Crabfarm
     end
 
     def unload_services
-      release_driver_pool
       release_http_client
-      release_phantom
-    end
-
-    def init_driver_pool
-      @pool = DriverBucketPool.new build_driver_factory if @pool.nil?
+      release_driver_pool
+      release_driver_factory
     end
 
-    def release_driver_pool
-      @pool.release unless @pool.nil?
-      @pool = nil
+    def init_driver_factory
+      if @factory.nil?
+        @factory = Strategies.load(:browser, config.browser).new proxy
+        @factory.prepare_driver_services
+      end
     end
 
-    def init_phantom_if_required
-      if config.phantom_mode_enabled? and @phantom.nil?
-        @phantom = load_and_start_phantom
-      end
+    def release_driver_factory
+      @factory.cleanup_driver_services unless @factory.nil?
+      @factory.nil?
     end
 
-    def load_and_start_phantom
-      phantom_port = Utils::PortDiscovery.find_available_port
-      new_phantom = PhantomRunner.new phantom_config.merge(port: phantom_port)
-      new_phantom.start
-      return new_phantom
+    def init_driver_pool
+      @pool = DriverPool.new @factory if @pool.nil?
     end
 
-    def release_phantom
-      @phantom.stop unless @phantom.nil?
-      @phantom = nil
+    def release_driver_pool
+      @pool.release unless @pool.nil?
+      @pool = nil
     end
 
     def init_http_client
-      @http = build_http_client if @http.nil?
+      @http = HttpClient.new proxy if @http.nil?
     end
 
     def release_http_client
       @http = nil
     end
 
-    def build_driver_factory
-      if @phantom
-        PhantomDriverFactory.new @phantom, driver_config
-      else
-        return config.driver_factory if config.driver_factory
-        DefaultDriverFactory.new driver_config
-      end
-    end
-
-    def build_http_client
-      HttpClient.new config.proxy
+    def proxy
+      Crabfarm.config.proxy
     end
 
     def config
       Crabfarm.config
     end
 
-    def driver_config
-      config.driver_config
-    end
-
-    def phantom_config
-      config.phantom_config
-    end
-
   end
-
 end

data/lib/crabfarm/crabtrap_context.rb

@@ -57,10 +57,6 @@ module Crabfarm
       @port = nil
     end
 
-    def build_http_client
-      HttpClient.new proxy_address
-    end
-
     def start_daemon
       if @runner.nil?
         options = {
@@ -69,7 +65,7 @@ module Crabfarm
           port: @port
         }
 
-        @runner = CrabtrapRunner.new Crabfarm.config.crabtrap_config.merge(options)
+        @runner = CrabtrapRunner.new config.crabtrap_config.merge(options)
         @runner.start
       end
     end
@@ -81,12 +77,9 @@ module Crabfarm
       else nil end
     end
 
-    def driver_config
-      super.merge(proxy: proxy_address)
-    end
-
-    def phantom_config
-      super.merge(proxy: proxy_address)
+    def proxy
+      # just step over configuration proxy
+      proxy_address
     end
 
     def proxy_address