crabfarm 0.0.6 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 591615890c50a205b34d45a201987d95f4ae3fae
-  data.tar.gz: f67f0538057b4708da26bd00a52ef4bcfaa7bd34
+  metadata.gz: 2c6a40d52475ec34390f4dca75207062dfdae26b
+  data.tar.gz: 9b298a6d3b4eb50d42eb153c274b638dff02ed63
 SHA512:
-  metadata.gz: ee2b73e89eef91889cdeb754dfe527fcbae9e4f2491e1d3c71169d5bcee7e2f0e42c1797a2299a756af24c87a74756f85c298354e009622395a0d218e83f5212
-  data.tar.gz: 4a0aee39da4bb582b025d417da92a57ba9269f1f5baf287c121369106494e4248cd761a121f662ea09bbe0acaa38d000a37d2e4049bcffed05f871cb1cc6a662
+  metadata.gz: f2ecc793d6ec88e0dba2cbd806195fc1c94785cefcea02a4384ca0a7a2e5fd5df983e35bf9a1488b8383251122191a30d6222f2a5796f259bcdf009bd3a4a8ef
+  data.tar.gz: 65763873d5038ad0eacb535732c9f0e395f91959b5328eb1726d3bd3e4aa7ee3af77bd5d6f93e398b4a60406f099a89e5952e61ff49e7d7d133524551e67a274

data/lib/crabfarm.rb

@@ -5,7 +5,7 @@ require "selenium-webdriver"
 require "crabfarm/version"
 require "crabfarm/errors"
 require "crabfarm/configuration"
-require "crabfarm/module_helper"
+require "crabfarm/loader_service"
 require "crabfarm/driver_bucket"
 require "crabfarm/driver_bucket_pool"
 require "crabfarm/default_driver_factory"
@@ -16,9 +16,19 @@ require "crabfarm/context"
 require "crabfarm/base_state"
 require "crabfarm/base_parser"
 require "crabfarm/strategies"
-require "crabfarm/loader"
 
 module Crabfarm
+
+  @@config = Configuration.new
+
+  def self.config
+    @@config
+  end
+
+  def self.read_crabfile(_path)
+    @@config.instance_eval File.read _path
+  end
+
   module Strategies
     # bundled browser dsl adapters
     register :browser_dsl, :surfer, 'Crabfarm::SurferBrowserDsl', 'crabfarm/adapters/browser/surfer'

data/lib/crabfarm/base_parser.rb

@@ -7,8 +7,8 @@ module Crabfarm
       @browser_dsl = _dsl
     end
 
-    def initialize(_module, _driver, _params)
-      dsl_class = Strategies.load(:browser_dsl, class_browser_dsl || _module.settings.browser_dsl)
+    def initialize(_driver, _params)
+      dsl_class = Strategies.load(:browser_dsl, class_browser_dsl || Crabfarm.config.browser_dsl)
       @browser = dsl_class.wrap _driver
       @params = _params
 

data/lib/crabfarm/base_state.rb

@@ -18,14 +18,13 @@ module Crabfarm
       @class_output_builder = _builder
     end
 
-    def initialize(_module, _pool, _store, _params)
-      @module = _module
+    def initialize(_pool, _store, _params)
       @pool = _pool
       @store = _store
       @params = _params
 
-      @dsl = Strategies.load(:browser_dsl, class_browser_dsl || @module.settings.browser_dsl)
-      @builder = Strategies.load(:output_builder, class_output_builder || @module.settings.output_builder)
+      @dsl = Strategies.load(:browser_dsl, class_browser_dsl || Crabfarm.config.browser_dsl)
+      @builder = Strategies.load(:output_builder, class_output_builder || Crabfarm.config.output_builder)
     end
 
     def browser(_name=nil)

data/lib/crabfarm/cli.rb

@@ -13,11 +13,11 @@ module Crabfarm
       Support::GLI.generate_options c
 
       c.action do |global_options,options,args|
-        next puts "This command can only be run inside a crabfarm application" unless defined? CF_LOADER
+        next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
 
         require "crabfarm/modes/console"
-        CF_LOADER.load Support::GLI.parse_options options
-        Crabfarm::Modes::Console.start CF_LOADER
+        Crabfarm.config.set Support::GLI.parse_options options
+        Crabfarm::Modes::Console.start
       end
     end
 
@@ -32,10 +32,13 @@ module Crabfarm
       c.desc "Set the server min and max threads, defaults to 0:16"
       c.flag [:t,:threads]
 
+      c.desc "Activate code reload before every request"
+      c.switch :reload, :default_value => true
+
       Support::GLI.generate_options c
 
       c.action do |global_options,options,args|
-        next puts "This command can only be run inside a crabfarm application" unless defined? CF_LOADER
+        next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
 
         require "crabfarm/modes/server"
         server_options = {}
@@ -43,8 +46,9 @@ module Crabfarm
         server_options[:Port] = options[:port] || 3100
         server_options[:Threads] = options[:threads] unless options[:threads].nil?
 
-        CF_LOADER.load Support::GLI.parse_options options
-        Crabfarm::Modes::Server.start CF_LOADER, server_options
+        ActiveSupport::Dependencies.mechanism = :require unless options[:reload]
+        Crabfarm.config.set Support::GLI.parse_options options
+        Crabfarm::Modes::Server.start server_options
       end
     end
 

data/lib/crabfarm/configuration.rb

@@ -39,6 +39,10 @@ module Crabfarm
     end
 
     def initialize
+      reset
+    end
+
+    def reset
       @values = {
         browser_dsl: :surfer,
         output_builder: :hash,

data/lib/crabfarm/context.rb

@@ -6,14 +6,13 @@ module Crabfarm
 
     def_delegators :@pool, :driver
 
-    def initialize(_module)
-      @module = _module
-      @pool = DriverBucketPool.new @module
-      @store = StateStore.new @module
+    def initialize
+      @pool = DriverBucketPool.new
+      @store = StateStore.new
     end
 
     def run_state(_name, _params={})
-      state = @module.load_state(_name).new @module, @pool, @store, _params
+      state = LoaderService.load_state(_name).new @pool, @store, _params
       state.crawl
       state
     end

data/lib/crabfarm/default_driver_factory.rb

@@ -11,6 +11,9 @@ module Crabfarm
       raise ConfigurationError.new 'must provide a webdriver type' if driver_name.nil?
 
       case driver_name
+      when :noop
+        require "crabfarm/mocks/noop_driver"
+        driver = Crabfarm::Mocks::NoopDriver.new # TODO: improve dummy driver...
       when :remote
         # setup a custom client to use longer timeouts
         client = Selenium::WebDriver::Remote::Http::Default.new

data/lib/crabfarm/driver_bucket.rb

@@ -3,8 +3,7 @@ module Crabfarm
 
     attr_reader :session_id
 
-    def initialize(_module, _session_id, _factory)
-      @module = _module
+    def initialize(_session_id, _factory)
       @session_id = _session_id
       @factory = _factory
       @driver = nil
@@ -16,8 +15,8 @@ module Crabfarm
     end
 
     def parse(_parser_class, _options={})
-      _parser_class = @module.load_parser(_parser_class) if _parser_class.is_a? String or _parser_class.is_a? Symbol
-      parser = _parser_class.new @module, self, _options
+      _parser_class = LoaderService.load_parser(_parser_class) if _parser_class.is_a? String or _parser_class.is_a? Symbol
+      parser = _parser_class.new self, _options
       parser.parse
       return parser
     end

data/lib/crabfarm/driver_bucket_pool.rb

@@ -1,8 +1,7 @@
 module Crabfarm
   class DriverBucketPool
 
-    def initialize(_module)
-      @module = _module
+    def initialize
       @buckets = Hash.new
       @phantom = nil
 
@@ -12,7 +11,7 @@ module Crabfarm
     def driver(_session_id=nil)
       _session_id ||= :default_driver
       bucket = @buckets[_session_id.to_sym]
-      bucket = @buckets[_session_id.to_sym] = DriverBucket.new(@module, _session_id, build_driver_factory) if bucket.nil?
+      bucket = @buckets[_session_id.to_sym] = DriverBucket.new(_session_id, build_driver_factory) if bucket.nil?
       bucket
     end
 
@@ -29,20 +28,24 @@ module Crabfarm
   private
 
     def init_phantom_if_required
-      if @module.settings.phantom_mode_enabled?
-        @phantom = PhantomRunner.new @module.settings.phantom_config
+      if config.phantom_mode_enabled?
+        @phantom = PhantomRunner.new config.phantom_config
         @phantom.start
       end
     end
 
     def build_driver_factory
-      if @module.settings.phantom_mode_enabled?
-        PhantomDriverFactory.new @phantom, @module.settings.driver_config
+      if config.phantom_mode_enabled?
+        PhantomDriverFactory.new @phantom, config.driver_config
       else
-        return @module.settings.driver_factory if @module.settings.driver_factory
-        DefaultDriverFactory.new @module.settings.driver_config
+        return config.driver_factory if config.driver_factory
+        DefaultDriverFactory.new config.driver_config
       end
     end
 
+    def config
+      Crabfarm.config
+    end
+
   end
 end

data/lib/crabfarm/engines/safe_state_loop.rb

@@ -1,11 +1,12 @@
+require 'benchmark'
 require 'ostruct'
 
 module Crabfarm
   module Engines
     class SafeStateLoop
 
-      def initialize(_loader)
-        @context = _loader.load_context
+      def initialize
+        @context = Crabfarm::Context.new
         @running = true
         @working = false
         @lock = Mutex.new
@@ -21,6 +22,7 @@ module Crabfarm
       def change_state(_name, _params={}, _wait=nil)
         @lock.synchronize {
           raise StillWorkingError.new if @working
+          # TODO: test class reloading here?
           @next_state_name = _name
           @next_state_params = _params
           @working = true
@@ -69,20 +71,24 @@ module Crabfarm
         OpenStruct.new({
           name: @state_name,
           params: @state_params,
-          doc: @doc
+          doc: @doc,
+          elapsed: @elapsed
         })
       end
 
       def crawl_loop
         while @running
           if @working
-            begin
-              @doc = @context.run_state(@next_state_name, @next_state_params).output_as_json
-              @error = nil
-            rescue Exception => e
-              @doc = nil
-              @error = e
-            end
+            @elapsed = Benchmark.measure do
+              begin
+                ActiveSupport::Dependencies.clear
+                @doc = @context.run_state(@next_state_name, @next_state_params).output_as_json
+                @error = nil
+              rescue Exception => e
+                @doc = nil
+                @error = e
+              end
+            end.real
 
             @state_name = @next_state_name
             @state_params = @next_state_params

data/lib/crabfarm/loader_service.rb

@@ -0,0 +1,29 @@
+module Crabfarm
+  class LoaderService
+
+    def self.load_state(_name)
+      load_entity _name, 'state', BaseState
+    end
+
+    def self.load_parser(_name)
+      load_entity _name, 'parser', BaseParser
+    end
+
+  private
+
+    def self.load_entity(_name, _role, _type)
+
+      if _name.is_a? String or _name.is_a? Symbol
+        name = _name.to_s.gsub(/[^A-Z0-9:]+/i, '_').camelize
+        mod = name.constantize rescue nil
+      else
+        mod = _name
+      end
+
+      raise EntityNotFoundError.new _role, name if mod.nil?
+      raise EntityNotFoundError.new _role, name unless mod < _type
+      mod
+    end
+
+  end
+end

data/lib/crabfarm/mocks/noop_driver.rb

@@ -0,0 +1,6 @@
+module Crabfarm
+  module Mocks
+    class NoopDriver
+    end
+  end
+end

data/lib/crabfarm/modes/console.rb

@@ -1,3 +1,4 @@
+require 'benchmark'
 require 'readline'
 require 'rainbow'
 require 'rainbow/ext/string'
@@ -11,8 +12,7 @@ module Crabfarm
 
         attr_reader :context
 
-        def initialize(_loader)
-          @loader = _loader
+        def initialize
           reload!
         end
 
@@ -20,10 +20,10 @@ module Crabfarm
           unless @context.nil?
             puts "Reloading crawler source".color(:green)
             @context.release
-            @loader.reload_source
+            ActiveSupport::Dependencies.clear
           end
 
-          @context = @loader.load_context
+          @context = Crabfarm::Context.new
         end
 
         def transition(_name=nil, _params={})
@@ -33,8 +33,14 @@ module Crabfarm
           end
 
           begin
-            doc = @context.run_state(_name, _params).output_as_json
-            puts JSON.pretty_generate(doc).color(:green)
+            elapsed = Benchmark.measure do
+              puts "Transitioning to #{_name.to_s.camelize} state"
+              doc = @context.run_state(_name, _params).output_as_json
+
+              puts "State changed, generated document:"
+              puts JSON.pretty_generate(doc).color(:green).gsub(/(^|\\n)/, '  ')
+            end
+            puts "Completed in #{elapsed.real} s"
           rescue EntityNotFoundError => e
             puts "#{e.to_s}".color(:red)
           rescue => e
@@ -56,8 +62,8 @@ module Crabfarm
         alias :r :reset
       end
 
-      def self.start(_loader)
-        dsl = ConsoleDsl.new(_loader)
+      def self.start
+        dsl = ConsoleDsl.new
 
         loop do
           begin

data/lib/crabfarm/modes/generator.rb

@@ -33,16 +33,16 @@ module Crabfarm
       def generate_state(_name)
         with_crawler_path do
           binding = { state_class: _name.camelize }
-          path('app', 'states', _name.parameterize + '.rb').render('state.rb', binding)
-          path('spec', 'states', _name.parameterize + '_spec.rb').render('state_spec.rb', binding)
+          path('app', 'states', _name.underscore + '.rb').render('state.rb', binding)
+          path('spec', 'states', _name.underscore + '_spec.rb').render('state_spec.rb', binding)
         end
       end
 
       def generate_parser(_name)
         with_crawler_path do
           binding = { parser_class: _name.camelize }
-          path('app', 'parsers', _name.parameterize + '.rb').render('parser.rb', binding)
-          path('spec', 'parsers', _name.parameterize + '_spec.rb').render('parser_spec.rb', binding)
+          path('app', 'parsers', _name.underscore + '.rb').render('parser.rb', binding)
+          path('spec', 'parsers', _name.underscore + '_spec.rb').render('parser_spec.rb', binding)
         end
       end
 
@@ -77,18 +77,16 @@ module Crabfarm
         self
       end
 
-
-
   private
 
       def generate_dir(_path, _silent)
         path = File.join(*_path)
         dir = Pathname.new path
         unless dir.exist?
-          puts "Generating #{path}".color(:green)
+          render_op "mkdir", path, :green
           dir.mkpath
         else
-          puts "Skipping #{path}".color(:yellow) unless _silent
+          render_op "skip", path, :yellow unless _silent
         end
       end
 
@@ -97,13 +95,13 @@ module Crabfarm
         output = File.join(*_path)
 
         unless Pathname.new(output).exist?
-          puts "Rendering #{output}".color(:green)
+          render_op "render", output, :green
           File.open(output, "w") do |f|
             f.write eval_template_with_hash(template, _binding)
             f.chmod(_mod) unless _mod.nil?
           end
         else
-          puts "Skipping #{output}, already exists".color(:yellow)
+          render_op "skip", output, :yellow
         end
       end
 
@@ -115,6 +113,10 @@ module Crabfarm
       def template_dir
         File.expand_path('../../templates', __FILE__)
       end
+
+      def render_op(_op, _message, _color)
+        puts _op.rjust(10).color(_color) + '  ' + _message
+      end
     end
   end
 end

data/lib/crabfarm/modes/server.rb

@@ -34,7 +34,8 @@ module Crabfarm
             {
               name: _state.name,
               params: _state.params,
-              doc: _state.doc
+              doc: _state.doc,
+              elapsed: _state.elapsed
             }
           end
         end
@@ -61,8 +62,8 @@ module Crabfarm
         @@evaluator
       end
 
-      def self.start(_loader, _options)
-        @@evaluator = Engines::SafeStateLoop.new _loader
+      def self.start(_options)
+        @@evaluator = Engines::SafeStateLoop.new
         begin
           Support::CustomPuma.run API, _options
         ensure

data/lib/crabfarm/rspec.rb

@@ -1,6 +1,4 @@
-CF_LOADER.load
-
-CF_TEST_CONTEXT = CF_LOADER.load_context
+CF_TEST_CONTEXT = Crabfarm::Context::new
 CF_TEST_BUCKET = CF_TEST_CONTEXT.driver
 
 module Crabfarm

data/lib/crabfarm/state_store.rb

@@ -3,7 +3,7 @@ require 'active_support'
 module Crabfarm
   class StateStore
 
-    def initialize(_module)
+    def initialize
       reset
     end
 

data/lib/crabfarm/templates/boot.rb.erb

@@ -4,9 +4,13 @@ require 'bundler'
 
 ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __FILE__)
 Bundler.setup
-require 'crabfarm'
-
-# Run code loader
 
 CF_PATH = File.expand_path('../', __FILE__)
-CF_LOADER = Crabfarm::Loader.new CF_PATH
+
+# Setup code autoloading
+require 'active_support/dependencies'
+ActiveSupport::Dependencies.autoload_paths += Dir.glob File.join(CF_PATH, 'app', '**')
+
+# Load crabfarm framework and configuration
+require 'crabfarm'
+Crabfarm.read_crabfile File.join(CF_PATH, 'Crabfile')

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.0.6"
+  VERSION = "0.0.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.8
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-08 00:00:00.000000000 Z
+date: 2015-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jbuilder
@@ -310,11 +310,11 @@ files:
 - lib/crabfarm/engines/safe_state_loop.rb
 - lib/crabfarm/errors.rb
 - lib/crabfarm/forked_state.rb
-- lib/crabfarm/loader.rb
+- lib/crabfarm/loader_service.rb
+- lib/crabfarm/mocks/noop_driver.rb
 - lib/crabfarm/modes/console.rb
 - lib/crabfarm/modes/generator.rb
 - lib/crabfarm/modes/server.rb
-- lib/crabfarm/module_helper.rb
 - lib/crabfarm/phantom_driver_factory.rb
 - lib/crabfarm/phantom_runner.rb
 - lib/crabfarm/rspec.rb

data/lib/crabfarm/loader.rb

@@ -1,97 +0,0 @@
-require 'active_support'
-
-module Crabfarm
-
-  class Loader
-
-    attr_reader :module
-
-    def initialize(_base_path, _module=Object)
-      @path = _base_path
-      @module = _module
-      @config = nil
-      @source_loaded = false
-    end
-
-    def load(_overrides=nil)
-      load_config _overrides
-      load_source
-    end
-
-    def load_config(_overrides=nil)
-      raise ConfigurationError.new 'Source already loaded, call unload_source first' if @source_loaded
-      raise ConfigurationError.new 'Crabfile not found' unless File.exists? crafile_path
-
-      @config = read_crabfile crafile_path
-      @config.set _overrides unless _overrides.nil?
-    end
-
-    def load_source
-      raise ConfigurationError.new 'Crabfile must be loaded first' if @config.nil?
-      raise ConfigurationError.new 'Source already loaded, call reload_source instead' if @source_loaded
-
-      require_from_path source_path
-      @source_loaded = true
-    end
-
-    def unload_source
-      # TODO: unload every class in a list
-      @source_loaded = false
-    end
-
-    def reload_source
-      unload_source if @source_loaded
-      load_source
-    end
-
-    def is_loaded?
-      @source_loaded
-    end
-
-    def load_context(_overrides={})
-      raise ConfigurationError.new 'Must load source first' unless @source_loaded
-      Context.new ModuleHelper.new @module, @config
-    end
-
-  private
-
-    def crafile_path
-      File.join @path, 'Crabfile'
-    end
-
-    def source_path
-      File.join @path, 'app'
-    end
-
-    def read_crabfile(_path)
-      config = Configuration.new
-      config.instance_eval File.read crafile_path
-      return config
-    end
-
-    def require_from_path(_src_path)
-      @module.module_eval do
-        # TODO: put every loaded class in a list, store last update or hash so next time is required
-        # it can be reloaded automatically.
-
-        pending = Dir.glob(File.join(_src_path, '**/*')).select { |f| f.end_with? '.rb' }.map { |f| f[0...-3] }
-
-        while pending.size > 0
-          new_pending = []
-          pending.each do |file|
-            begin
-              require file
-            rescue NameError => e
-              new_pending << file
-            end
-          end
-
-          require new_pending.first if new_pending.size == pending.size
-          pending = new_pending
-        end
-      end
-    end
-
-  end
-
-end

data/lib/crabfarm/module_helper.rb

@@ -1,36 +0,0 @@
-require 'active_support'
-
-module Crabfarm
-  class ModuleHelper
-
-    attr_reader :dsl
-
-    def initialize(_module, _config=nil)
-      @module = _module
-      @config = _config || @module::CF_CONFIG
-    end
-
-    def settings
-      @config
-    end
-
-    def load_state(_name)
-      load_entity _name, 'state', BaseState
-    end
-
-    def load_parser(_name)
-      load_entity _name, 'parser', BaseParser
-    end
-
-  private
-
-    def load_entity(_name, _role, _type)
-      name = _name.to_s.gsub(/[^A-Z0-9:]+/i, '_').camelize
-      mod = @module.const_get(name) rescue nil
-      raise EntityNotFoundError.new _role, name if mod.nil?
-      raise EntityNotFoundError.new _role, name unless mod < _type
-      mod
-    end
-
-  end
-end