crabfarm 0.0.2

data/lib/crabfarm/driver_bucket_pool.rb

@@ -0,0 +1,48 @@
+module Crabfarm
+  class DriverBucketPool
+
+    def initialize(_module)
+      @module = _module
+      @buckets = Hash.new
+      @phantom = nil
+
+      init_phantom_if_required
+    end
+
+    def driver(_session_id=nil)
+      _session_id ||= :default_driver
+      bucket = @buckets[_session_id.to_sym]
+      bucket = @buckets[_session_id.to_sym] = DriverBucket.new(@module, _session_id, build_driver_factory) if bucket.nil?
+      bucket
+    end
+
+    def reset
+      @buckets.values.each(&:reset)
+      @buckets = Hash.new
+    end
+
+    def release
+      reset
+      @phantom.stop unless @phantom.nil?
+    end
+
+  private
+
+    def init_phantom_if_required
+      if @module.settings.phantom_enabled?
+        @phantom = PhantomRunner.new @module.settings.phantom_config
+        @phantom.start
+      end
+    end
+
+    def build_driver_factory
+      if @module.settings.phantom_enabled?
+        PhantomDriverFactory.new @phantom, @module.settings.driver_config
+      else
+        return @module.settings.driver_factory if @module.settings.driver_factory
+        DefaultDriverFactory.new @module.settings.driver_config
+      end
+    end
+
+  end
+end

data/lib/crabfarm/dsl/surfer.rb

@@ -0,0 +1,22 @@
+require 'crabfarm/dsl/surfer/search_context'
+require 'crabfarm/dsl/surfer/surf_context'
+
+module Crabfarm
+  module Dsl
+    module Surfer
+
+      class Error < StandardError
+        attr_reader :source
+
+        def initialize(_message, _ctx)
+          super _message
+          @ctx = _ctx
+          @source = _ctx.root_context.page_source rescue nil # cache page source for future reference
+        end
+      end
+
+      class EmptySetError < Error; end
+      class WebdriverError < Error; end
+    end
+  end
+end

data/lib/crabfarm/dsl/surfer/search_context.rb

@@ -0,0 +1,134 @@
+module Crabfarm
+  module Dsl
+    module Surfer
+      class SearchContext
+        include Enumerable
+        extend Forwardable
+
+        TIMEOUT = 10.0 # Default timeout for waiting operations
+
+        def initialize(_elements, _parent)
+          @elements = _elements
+          @parent = _parent
+        end
+
+        # return the context's root context
+        def root_context
+          return @parent.root_context if @parent
+          self
+        end
+
+        # return the context's parent context
+        def parent_context
+          @parent
+        end
+
+        # forward read-only array methods to context
+        def_delegators :context, :each, :[], :length, :count, :empty?, :first, :last
+
+        # yield individual SearchContext for each element contained in this result
+        def explode(&_block)
+          return enum_for(__method__) if _block.nil?
+          context.each do |el|
+            _block.call SearchContext.new([el], self)
+          end
+        end
+
+        # searches for elements that match a given selector
+        def search(_selector=nil, _options={})
+          _options[:css] = _selector if _selector
+
+          wait_mode = _options.delete :wait
+          if wait_mode
+
+            # retrieve timeout
+            timeout = _options.delete :timeout
+            timeout = TIMEOUT if timeout.nil?
+
+            # use a selenium timeout
+            wait = Selenium::WebDriver::Wait.new(timeout: timeout)
+            wait.until do
+              new_elements = search_elements _options
+
+              # test wait condition
+              ok = case wait_mode
+              when :present then (new_elements.length > 0)
+              when :visible then (new_elements.length > 0 and new_elements.first.displayed?)
+              when :enabled then (new_elements.length > 0 and new_elements.first.displayed? and new_elements.first.enabled?)
+              when :not_present then (new_elements.length == 0)
+              when :not_visible then (not new_elements.any? { |e| e.displayed? })
+              else
+                raise SetupError.new "Invalid wait mode '#{wait_mode}'"
+              end
+
+              SearchContext.new new_elements, self if ok
+            end
+          else
+            SearchContext.new search_elements(_options), self
+          end
+        end
+
+        # clears and sends_keys to this context main element
+        def fill(_value)
+          raise EmptySetError.new('Cannot call \'fill\' on an empty set', self) if empty?
+          wrap_errors do
+            context.first.clear
+            context.first.send_keys _value
+          end
+        end
+
+        # Any methods missing are forwarded to the main element (first).
+        def method_missing(_method, *_args, &_block)
+          wrap_errors do
+            m = /^(.*)_all$/.match _method.to_s
+            if m then
+              return [] if empty?
+              context.map { |e| e.send(m[1], *_args, &_block) }
+            else
+              raise EmptySetError.new("Cannot call '#{_method}' on an empty set", self) if empty?
+              context.first.send(_method, *_args, &_block)
+            end
+          end
+        end
+
+        def respond_to?(_method, _include_all=false)
+          return true if super
+          m = /^.*_all$/.match _method.to_s
+          if m then
+            return true if empty?
+            context.first.respond_to? m[1], _include_all
+          else
+            return true if empty?
+            context.first.respond_to? _method, _include_all
+          end
+        end
+
+      private
+
+        # wrap every selenium errors that happen inside block.
+        def wrap_errors
+          begin
+            yield
+          rescue Selenium::WebDriver::Error::WebDriverError => e
+            raise WebdriverError.new e, self
+          end
+        end
+
+        # base filtering method, expands current context
+        def search_elements(_options)
+          wrap_errors do
+            context.inject([]) do |r, element|
+              r + element.find_elements(_options)
+            end
+          end
+        end
+
+        # returns the current context
+        def context
+          @elements
+        end
+
+      end
+    end
+  end
+end

data/lib/crabfarm/dsl/surfer/surf_context.rb

@@ -0,0 +1,58 @@
+module Crabfarm
+  module Dsl
+    module Surfer
+      class SurfContext < SearchContext
+        extend Forwardable
+
+        def_delegators :@bucket, :parse, :setup
+        def_delegators 'driver.navigate', :back, :forward, :refresh
+
+        def initialize(_bucket)
+          super nil, nil
+          @bucket = _bucket
+        end
+
+        def driver
+          @bucket.original
+        end
+
+        def quit
+          @bucket.reset
+        end
+
+        def current_uri
+          URI.parse driver.current_url
+        end
+
+        def cookies
+          driver.manage.all_cookies
+        end
+
+        def goto(_url, _params=nil)
+          _url += "?#{_params.to_query}" if _params
+          retries = 0
+
+          loop do
+            begin
+              @bucket.reset if retries > 0
+              driver.get(_url)
+              break
+            rescue Timeout::Error #, Selenium::WebDriver::Error::UnknownError
+              # TODO: log this
+              raise if retries >= max_retries
+              retries += 1
+              sleep 1.0
+            end
+          end
+        end
+
+      private
+
+        def context
+          [driver]
+        end
+
+      end
+    end
+  end
+end

data/lib/crabfarm/engines/safe_state_loop.rb

@@ -0,0 +1,96 @@
+require 'ostruct'
+
+module Crabfarm
+  module Engines
+    class SafeStateLoop
+
+      def initialize(_loader)
+        @context = _loader.load_context
+        @running = true
+        @working = false
+        @lock = Mutex.new
+        @thread = Thread.new { crawl_loop }
+      end
+
+      def release
+        @running = false
+        @thread.join
+        @context.release
+      end
+
+      def change_state(_name, _params={}, _wait=nil)
+        @lock.synchronize {
+          raise StillWorkingError.new if @working
+          @next_state_name = _name
+          @next_state_params = _params
+          @working = true
+
+          wait_and_load_struct _wait
+        }
+      end
+
+      def wait_for_state(_wait=nil)
+        @lock.synchronize {
+          wait_and_load_struct _wait
+        }
+      end
+
+      def cancel
+        @lock.synchronize {
+          if @working
+            @thread.terminate
+            @thread.join
+            @thread = Thread.new { crawl_loop }
+            @working = false
+          end
+        }
+      end
+
+    private
+
+      def wait_and_load_struct(_wait)
+        # need to use this method because mutex are not reentrant and monitors are slow.
+        wait_while_working _wait unless _wait.nil?
+        raise TimeoutError.new if @working
+        state_as_struct
+      end
+
+      def wait_while_working(_wait)
+        # TODO: use condition variables instead of wait loops
+        start = Time.now
+        while @working and Time.now - start < _wait.seconds
+          @lock.sleep 0.25
+        end
+      end
+
+      def state_as_struct
+        raise CrawlerError.new @error if @error
+
+        OpenStruct.new({
+          name: @state_name,
+          params: @state_params,
+          doc: @doc
+        })
+      end
+
+      def crawl_loop
+        while @running
+          if @working
+            begin
+              last_state = @context.run_state @next_state_name, @next_state_params
+              @doc = last_state.output.attributes!
+              @error = nil
+            rescue Exception => e
+              @doc = nil
+              @error = e
+            end
+
+            @state_name = @next_state_name
+            @state_params = @next_state_params
+            @working = false
+          else sleep 0.2 end
+        end
+      end
+    end
+  end
+end

data/lib/crabfarm/errors.rb

@@ -0,0 +1,50 @@
+module Crabfarm
+
+  class Error < StandardError; end
+
+  class ConfigurationError < Error; end
+
+  class EntityNotFoundError < Error
+    attr_accessor :role, :name
+
+    def initialize(_role, _name)
+      super("The required #{_role} was not found (#{_name})")
+      @role = _role
+      @name = _name
+    end
+  end
+
+  class ApiError < Error
+    def code; 500 end
+    def to_json; {} end
+  end
+
+  class StillWorkingError < ApiError
+    def code; 409 end
+  end
+
+  class TimeoutError < ApiError
+    def code; 408 end
+  end
+
+  class CrawlerBaseError < ApiError
+    def initialize(_msg, _trace)
+      @exc = _msg
+      @trace = _trace
+    end
+
+    def to_json
+      {
+        exception: @exc,
+        backtrace: @trace
+      }
+    end
+  end
+
+  class CrawlerError < CrawlerBaseError
+    def initialize(_exc)
+      super _exc.to_s, _exc.backtrace
+    end
+  end
+
+end

data/lib/crabfarm/loader.rb

@@ -0,0 +1,83 @@
+require 'active_support'
+
+module Crabfarm
+
+  class Loader
+
+    attr_reader :module
+
+    def initialize(_base_path, _module_name=nil, &_config_block)
+      @path = _base_path
+      @name = _module_name
+      @config_block = _config_block
+      @module = if is_wrapped? then
+        "::#{@name}".constantize rescue nil
+      else nil end
+    end
+
+    def is_wrapped?
+      @name.present?
+    end
+
+    def is_loaded?
+      not @module.nil?
+    end
+
+    def load
+      crabfile = load_crabfile(@path)
+      @module = load_module(@name, File.join(@path, 'app'))
+      @module.send(:remove_const, :CF_CONFIG) rescue nil
+      @module.const_set :CF_CONFIG, crabfile
+    end
+
+    def load_context
+      load unless is_loaded?
+      Context.new @module
+    end
+
+    def unload
+      Object.send(:remove_const, @name) if is_wrapped?
+      @module = nil
+    end
+
+  private
+
+    def load_crabfile(_path)
+      crabfile = File.read(File.join(_path, 'Crabfile'))
+      config = Configuration.new
+      config.instance_eval crabfile
+      @config_block.call(config) unless @config_block.nil?
+      return config
+    end
+
+    def load_module(_name, _path)
+      require_all_as(_name, _path)
+      if is_wrapped? then "::#{_name}".constantize else Object end
+    end
+
+    def require_all_as(_name, _src_path)
+      loader_code = "
+        pending = Dir.glob('#{File.join(_src_path, '**/*')}').select { |f| f.end_with? '.rb' }.map { |f| f[0...-3] }
+
+        while pending.size > 0
+          new_pending = []
+          pending.each do |file|
+            begin
+              require file
+            rescue NameError => e
+              new_pending << file
+            end
+          end
+
+          require new_pending.first if new_pending.size == pending.size
+          pending = new_pending
+        end
+      "
+
+      loader_code = "module ::#{_name}; #{loader_code}; end" if _name.present?
+      Object.instance_eval loader_code
+    end
+
+  end
+
+end