crabfarm 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49343d990ddd94c9640f37040a6fe5fc9a97ab49
-  data.tar.gz: f75e89f9dd4b5154f01df31837b98981fed38a3c
+  metadata.gz: c10577d223677baa807f27aa55fc8f2ccf19f189
+  data.tar.gz: 7d07015295fe0325a85d73396821872e0e3542d3
 SHA512:
-  metadata.gz: f841a5d9a50476e38b7a1252b3de75fc132ffcbc4ecb7bd25f49cbfad976eab75f825f47c5ca1afa9c41c53f4713baeb2da8578ed233b2be40897b25f859348f
-  data.tar.gz: c183db189f4968561409c0eea25a54a7ad4fb015b7d8cd3abcceefe3edc7790392626b5944b4bfa5d32b06bf074f357b00867762fb8ee3e1a97889cb0618e778
+  metadata.gz: 45015ef058a82381fc1db17791bb99c362c37c8bf0cf24e29d0862c76c44f006c47e60f6d2689b545be7ec564ce05857565ca7bc00d43f592cba0f58a58f658e
+  data.tar.gz: 83dc7148c8ed7b793290b0c197ad6ad10b1853d196d5c650a471cd4a26531d424177a303d82a25a4780a806d0b075683234318bef9f10f0ed1283f03f807d4df

data/lib/crabfarm.rb

@@ -7,7 +7,7 @@ require "selenium-webdriver"
 require "crabfarm/version"
 require "crabfarm/errors"
 require "crabfarm/configuration"
-require "crabfarm/transition_service"
+require "crabfarm/global_state"
 require "crabfarm/driver_bucket"
 require "crabfarm/driver_bucket_pool"
 require "crabfarm/http_client"
@@ -16,10 +16,14 @@ require "crabfarm/phantom_driver_factory"
 require "crabfarm/phantom_runner"
 require "crabfarm/state_store"
 require "crabfarm/context"
+require "crabfarm/context_factory"
+require "crabfarm/transition_service"
 require "crabfarm/base_state"
 require "crabfarm/base_parser"
 require "crabfarm/strategies"
 
+require "crabfarm/utils/port_discovery"
+
 module Crabfarm
 
   @@config = Configuration.new

data/lib/crabfarm/cli.rb

@@ -10,28 +10,20 @@ module Crabfarm
     desc "Starts the crawler in console mode"
     command [:console, :c] do |c|
 
-      c.desc "Capture to crabtrap file"
-      c.flag :capture
-
-      c.desc "Replay from crabtrap file"
-      c.flag :replay
+      c.desc "Use a recorded memento as data source, requires crabtrap"
+      c.flag [:m, :memento]
 
       Support::GLI.generate_options c
 
       c.action do |global_options,options,args|
-        next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
+        next puts "This command can only be run inside a crabfarm application" unless GlobalState.inside_crawler_app?
 
         Crabfarm.config.set Support::GLI.parse_options options
 
-        next puts "Cannot use --replay with --capture" if options[:capture] and options[:replay]
-
-        require 'crabfarm/crabtrap_context'
-        context = Crabfarm::CrabtrapContext.new
-        context.capture options[:capture] if options[:capture]
-        context.replay options[:replay] if options[:replay]
-
-        require "crabfarm/modes/console"
-        Crabfarm::Modes::Console.start context
+        ContextFactory.with_context options[:memento] do |context|
+          require "crabfarm/modes/console"
+          Crabfarm::Modes::Console.process_input context
+        end
       end
     end
 
@@ -46,6 +38,9 @@ module Crabfarm
       c.desc "Set the server min and max threads, defaults to 0:16"
       c.flag [:t,:threads]
 
+      c.desc "Use a recorded memento as data source, requires crabtrap"
+      c.flag [:m, :memento]
+
       c.desc "Start the server in verbose mode"
       c.switch :verbose, :default_value => false
 
@@ -55,18 +50,22 @@ module Crabfarm
       Support::GLI.generate_options c
 
       c.action do |global_options,options,args|
-        next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
+        next puts "This command can only be run inside a crabfarm application" unless GlobalState.inside_crawler_app?
+
+        Crabfarm.config.set Support::GLI.parse_options options
+
+        ActiveSupport::Dependencies.mechanism = :require unless options[:reload]
 
-        require "crabfarm/modes/server"
         server_options = {}
         server_options[:Host] = options[:host] unless options[:host].nil?
         server_options[:Port] = options[:port] || 3100
         server_options[:Threads] = options[:threads] unless options[:threads].nil?
         server_options[:Verbose] = options[:verbose]
 
-        ActiveSupport::Dependencies.mechanism = :require unless options[:reload]
-        Crabfarm.config.set Support::GLI.parse_options options
-        Crabfarm::Modes::Server.start server_options
+        ContextFactory.with_context options[:memento] do |context|
+          require "crabfarm/modes/server"
+          Crabfarm::Modes::Server.serve context, server_options
+        end
       end
     end
 
@@ -79,6 +78,8 @@ module Crabfarm
         app.flag [:r, :remote]
 
         app.action do |global_options,options,args|
+          next puts "This command cannot be run inside a crabfarm application" if GlobalState.inside_crawler_app?
+
           require "crabfarm/modes/generator"
           Crabfarm::Modes::Generator.generate_app(Dir.pwd, args[0], options[:remote])
         end
@@ -87,27 +88,34 @@ module Crabfarm
       c.desc "Generates a new crabfarm parser and parser spec"
       c.command :parser do |parser|
         parser.action do |global_options,options,args|
+          next puts "This command can only be run inside a crabfarm application" unless GlobalState.inside_crawler_app?
+
           require "crabfarm/modes/generator"
-          Crabfarm::Modes::Generator.generate_parser(args[0])
+          Crabfarm::Modes::Generator.generate_parser(GlobalState.app_path, args[0])
         end
       end
 
       c.desc "Generates a new crabfarm state and parser spec"
       c.command :state do |parser|
         parser.action do |global_options,options,args|
+          next puts "This command can only be run inside a crabfarm application" unless GlobalState.inside_crawler_app?
+
           require "crabfarm/modes/generator"
-          Crabfarm::Modes::Generator.generate_state(args[0])
+          Crabfarm::Modes::Generator.generate_state(GlobalState.app_path, args[0])
         end
       end
     end
 
     desc "Perform an HTTP recording for use in tests"
     command [:record, :r] do |c|
+      c.desc "Run recorder in playback mode"
+      c.switch [:p, :playback], :default_value => false
+
       c.action do |global_options, options, args|
-        next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
+        next puts "This command can only be run inside a crabfarm application" unless GlobalState.inside_crawler_app?
 
         require "crabfarm/modes/recorder"
-        Crabfarm::Modes::Recorder.start args[0]
+        Crabfarm::Modes::Recorder.start GlobalState.memento_path(args[0]), options[:playback]
       end
     end
 
@@ -120,12 +128,27 @@ module Crabfarm
       c.switch :unsafe, :default_value => false
 
       c.action do |global_options,options,args|
-        next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
+        next puts "This command can only be run inside a crabfarm application" unless GlobalState.inside_crawler_app?
 
         options[:remote] = args[0]
 
         require "crabfarm/modes/publisher"
-        Crabfarm::Modes::Publisher.publish CF_PATH, options
+        Crabfarm::Modes::Publisher.publish GlobalState.app_path, options
+      end
+    end
+
+    on_error do |exc|
+      case exc
+      when BinaryMissingError
+        if exc.binary == 'phantomjs'
+          puts "Could not find the phantomjs binary at '#{exc.path}', try installing it using 'npm install phantomjs -g' or set the propper path in your project's Crabfile"
+          false
+        elsif exc.binary == 'crabtrap'
+          puts "Could not find the crabtrap binary at '#{exc.path}', try installing it using 'npm install crabtrap -g' or set the propper path in your project's Crabfile"
+          false
+        else true end
+      else
+        true
       end
     end
 

data/lib/crabfarm/configuration.rb

@@ -25,12 +25,12 @@ module Crabfarm
       [:phantom_load_images, :boolean, 'Phantomjs image loading, only for phantomjs driver.'],
       [:phantom_ssl, ['sslv3', 'sslv2', 'tlsv1', 'any'], 'Phantomjs ssl mode: sslv3, sslv2, tlsv1 or any, only for phantomjs driver.'],
       [:phantom_bin_path, :string, 'Phantomjs binary path, only for phantomjs driver.'],
-      [:phantom_lock_file, :string, 'Phantomjs lock file path, only for phantomjs driver.'],
 
       # Crabtrap launcher configuration
       [:crabtrap_bin_path, :string, 'Crabtrap binary path.'],
-      [:crabtrap_port, :integer, 'Crabtrap port, defaults to 4000.'],
-      [:crabtrap_mode, ['capture', 'replay'], 'Crabtrap operation mode.']
+
+      # Recorder configuration
+      [:recorder_driver, :string, 'Recorder driver name, defaults to \'firefox\'']
     ]
     .map { |o| Option.new *o }
 
@@ -56,7 +56,6 @@ module Crabfarm
         driver_factory: nil,
         log_path: nil,
         proxy: nil,
-
         driver: 'phantomjs',
         driver_capabilities: Selenium::WebDriver::Remote::Capabilities.firefox,
         driver_host: 'localhost',
@@ -64,14 +63,11 @@ module Crabfarm
         driver_remote_timeout: 120,
         driver_window_width: 1280,
         driver_window_height: 800,
-
         phantom_load_images: false,
         phantom_ssl: 'any',
         phantom_bin_path: 'phantomjs',
-        phantom_lock_file: nil,
-
         crabtrap_bin_path: 'crabtrap',
-        crabtrap_port: 4000
+        recorder_driver: :firefox
       }
     end
 
@@ -108,7 +104,6 @@ module Crabfarm
         proxy: proxy,
         ssl: phantom_ssl,
         bin_path: phantom_bin_path,
-        lock_file: phantom_lock_file,
         log_file: log_path ? File.join(log_path, 'phantom.log') : nil
       }
     end
@@ -116,7 +111,6 @@ module Crabfarm
     def crabtrap_config
       {
         bin_path: crabtrap_bin_path,
-        port: crabtrap_port,
         proxy: proxy
       }
     end

data/lib/crabfarm/context.rb

@@ -11,26 +11,45 @@ module Crabfarm
       @loaded = false
     end
 
-    def load
+    def loaded?
+      @loaded
+    end
+
+    def prepare
+      unless @loaded
+        load_services
+        @loaded = true
+      end
+    end
+
+    def reset
+      reset_services if @loaded
+    end
+
+    def release
+      unload_services
+      @loaded = false
+    end
+
+  private
+
+    def load_services
       init_phantom_if_required
       init_driver_pool
       init_http_client
-      @loaded = true
     end
 
-    def reset
+    def reset_services
       @store.reset
-      @pool.reset unless @pool.nil?
+      @pool.reset
     end
 
-    def release
+    def unload_services
       release_driver_pool
+      release_http_client
       release_phantom
-      @loaded = false
     end
 
-  private
-
     def init_driver_pool
       @pool = DriverBucketPool.new build_driver_factory if @pool.nil?
     end
@@ -47,7 +66,8 @@ module Crabfarm
     end
 
     def load_and_start_phantom
-      new_phantom = PhantomRunner.new phantom_config
+      phantom_port = Utils::PortDiscovery.find_available_port
+      new_phantom = PhantomRunner.new phantom_config.merge(port: phantom_port)
       new_phantom.start
       return new_phantom
     end

data/lib/crabfarm/context_factory.rb

@@ -0,0 +1,31 @@
+module Crabfarm
+  module ContextFactory
+
+    def with_context(_memento=nil)
+      ctx = build_context(_memento)
+      begin
+        ctx.prepare
+        yield ctx
+      ensure
+        ctx.release
+      end
+    end
+
+    def build_context(_memento=nil)
+      if _memento.nil?
+        Crabfarm::Context.new
+      else
+        load_crabtrap_context _memento
+      end
+    end
+
+    def load_crabtrap_context(_memento)
+      require 'crabfarm/crabtrap_context'
+      m_path = GlobalState.memento_path _memento
+      raise ResourceNotFoundError.new "Could not find memento '#{_name}'" unless File.exists? m_path
+      Crabfarm::CrabtrapContext.new :replay, m_path
+    end
+
+    extend self
+  end
+end

data/lib/crabfarm/crabtrap_context.rb

@@ -4,54 +4,94 @@ require 'crabfarm/crabtrap_runner'
 module Crabfarm
   class CrabtrapContext < Context
 
-    def load
-      pass_through if @runner.nil?
-      super
+    attr_accessor :mode
+
+    def initialize(_mode=:pass, _path=nil)
+      @mode = _mode
+      @path = _path
     end
 
     def pass_through
-      restart_with_options(mode: :pass) if @runner.nil? or @runner.mode != :pass
+      if not loaded? or @mode != :pass
+        @mode = :pass
+        @path = nil
+        restart
+      end
     end
 
     def capture(_path)
-      restart_with_options(mode: :capture, bucket_path: _path)
+      @mode = :capture
+      @path = _path
+      restart
     end
 
     def replay(_path)
-      restart_with_options(mode: :replay, bucket_path: _path)
+      @mode = :replay
+      @path = _path
+      restart
     end
 
-    def release
-      super
-      stop_daemon
+    def restart
+      if not loaded?
+        prepare
+      else
+        stop_daemon
+        start_daemon
+      end
     end
 
   private
 
+    def load_services
+      @port = Utils::PortDiscovery.find_available_port
+      start_daemon
+      super
+    end
+
+    def reset_services
+      restart
+    end
+
+    def unload_services
+      super
+      stop_daemon
+      @port = nil
+    end
+
     def build_http_client
       HttpClient.new proxy_address
     end
 
-    def restart_with_options(_options)
-      stop_daemon
-      @runner = CrabtrapRunner.new Crabfarm.config.crabtrap_config.merge(_options)
-      @runner.start
+    def start_daemon
+      if @runner.nil?
+        options = {
+          mode: @mode,
+          bucket_path: @path,
+          port: @port
+        }
+
+        @runner = CrabtrapRunner.new Crabfarm.config.crabtrap_config.merge(options)
+        @runner.start
+      end
     end
 
     def stop_daemon
-      @runner.stop unless @runner.nil?
+      unless @runner.nil?
+        @runner.stop
+        @runner = nil
+      else nil end
     end
 
     def driver_config
-      if @runner.is_running? then super.merge(proxy: proxy_address) else super end
+      super.merge(proxy: proxy_address)
     end
 
     def phantom_config
-      if @runner.is_running? then super.merge(proxy: proxy_address) else super end
+      super.merge(proxy: proxy_address)
     end
 
     def proxy_address
-      "127.0.0.1:#{@runner.port}"
+      "127.0.0.1:#{@port}"
     end
 
   end

data/lib/crabfarm/crabtrap_runner.rb

@@ -1,6 +1,10 @@
+require 'timeout'
+
 module Crabfarm
   class CrabtrapRunner
 
+    CRABTRAP_START_TM = 5 # seconds
+
     def initialize(_config={})
       @config = _config;
       @pid = nil
@@ -11,7 +15,7 @@ module Crabfarm
     end
 
     def port
-      @config[:port] # TODO: maybe select port dynamically...
+      @config[:port]
     end
 
     def mode
@@ -19,25 +23,39 @@ module Crabfarm
     end
 
     def start
-      begin
-        @pid = Process.spawn({}, crabtrap_cmd)
-        wait_for_server
-      rescue
-        puts "Could not find crabtrap at #{@config[:bin_path]}, memento replaying is disabled!"
-        @pid = nil
-      end
+      logger.info "Starting crabtrap in port #{port}"
+      @pid = spawn_crabtrap
+      logger.info "Crabtrap started (PID: #{@pid})"
     end
 
     def stop
       unless @pid.nil?
+        logger.info "Stopping crabtrap (PID: #{@pid})"
         Process.kill("INT", @pid)
         Process.wait @pid
+        logger.info "Crabtrap stopped"
         @pid = nil
       end
     end
 
   private
 
+    def spawn_crabtrap
+      pid = nil
+      begin
+        pid = Process.spawn({}, crabtrap_cmd)
+        Timeout::timeout(CRABTRAP_START_TM) { wait_for_server }
+        return pid
+      rescue Errno::ENOENT
+        raise BinaryMissingError.new 'crabtrap', @config[:bin_path]
+      rescue Timeout::Error
+        Process.kill "INT", pid
+        Process.wait pid
+        raise
+      end
+      pid
+    end
+
     def crabtrap_cmd
       cmd = [@config[:bin_path]]
       cmd << mode.to_s
@@ -57,5 +75,9 @@ module Crabfarm
       end
     end
 
+    def logger
+      Crabfarm.logger
+    end
+
   end
 end

data/lib/crabfarm/engines/{safe_state_loop.rb → async_state_manager.rb}

@@ -3,11 +3,12 @@ require 'ostruct'
 
 module Crabfarm
   module Engines
-    class SafeStateLoop
+    class AsyncStateManager
 
       class LoopAbortedException < StandardError; end
 
-      def initialize
+      def initialize(_context)
+        @context = _context
         @working = false
         @fatal = nil
         @lock = Mutex.new
@@ -37,7 +38,7 @@ module Crabfarm
         start
       end
 
-      def change_state(_name, _params={}, _wait=nil)
+      def transition(_name, _params={}, _wait=nil)
         @lock.synchronize {
           if @fatal
             raise CrawlerError.new @fatal
@@ -100,25 +101,24 @@ module Crabfarm
       end
 
       def crawl_loop
-        context = Crabfarm::Context.new
-
         begin
           loop do
             if @working
-              @elapsed = Benchmark.measure do
-                begin
+              begin
+                logger.info "Transitioning state: #{@next_state_name}"
+                @elapsed = Benchmark.measure do
                   ActiveSupport::Dependencies.clear
-                  logger.info "StateLoop: loading state: #{@next_state_name}"
-                  @doc = TransitionService.apply_state(context, @next_state_name, @next_state_params).output_as_json
-                  logger.info "StateLoop: state loaded successfully: #{@next_state_name}"
-                  @error = nil
-                rescue Exception => e
-                  logger.error "StateLoop: error while loading state: #{@next_state_name}"
-                  logger.error e
-                  @doc = nil
-                  @error = e
-                end
-              end.real
+                  @doc = TransitionService.apply_state(@context, @next_state_name, @next_state_params).output_as_json
+                end.real
+
+                logger.info "Transitioned in #{@elapsed.real}"
+                @error = nil
+              rescue Exception => e
+                logger.error "Error during transition:"
+                logger.error e
+                @doc = nil
+                @error = e
+              end
 
               @lock.synchronize {
                 @state_name = @next_state_name
@@ -128,17 +128,15 @@ module Crabfarm
             else sleep 0.2 end
           end
         rescue LoopAbortedException
-          logger.info "StateLoop: stopping"
+          logger.info "Manager stopping"
 
         rescue Exception => e
-          logger.fatal "StateLoop: unhandled exception!"
+          logger.fatal "Unhandled exception:"
           logger.fatal e
 
           @lock.synchronize {
             @fatal = e
           }
-        ensure
-          context.release
         end
       end
 

data/lib/crabfarm/engines/sync_state_manager.rb

@@ -0,0 +1,40 @@
+require 'benchmark'
+require 'ostruct'
+
+module Crabfarm
+  module Engines
+    class SyncStateManager
+
+      def initialize(_context)
+        @context = _context
+        @lock = Mutex.new
+      end
+
+      def reload!
+        @lock.synchronize {
+          ActiveSupport::Dependencies.clear
+          @context.reset
+        }
+      end
+
+      def reset
+        @lock.synchronize {
+          @context.reset
+        }
+      end
+
+      def transition(_name, _params={})
+        @lock.synchronize {
+          output = { name: _name, params: _params }
+
+          output[:elapsed] = Benchmark.measure do
+            output[:doc] = TransitionService.apply_state(@context, _name, _params).output_as_json
+          end
+
+          OpenStruct.new output
+        }
+      end
+
+    end
+  end
+end

data/lib/crabfarm/errors.rb

@@ -4,18 +4,25 @@ module Crabfarm
 
   class ConfigurationError < Error; end
 
-  class AssertionError < Error; end
+  class BinaryMissingError < ConfigurationError
 
-  class EntityNotFoundError < Error
-    attr_accessor :role, :name
+    attr_accessor :binary
+    attr_accessor :path
 
-    def initialize(_role, _name)
-      super("The required #{_role} was not found (#{_name})")
-      @role = _role
-      @name = _name
+    def initialize(_binary, _path)
+      @binary = _binary
+      @path = _path
+      super "Could not find a suitable version of #{@binary}"
     end
+
   end
 
+  class AssertionError < Error; end
+
+  class ArgumentError < Error; end
+
+  class ResourceNotFoundError < Crabfarm::Error; end
+
   class ApiError < Error
     def code; 500 end
     def to_json; {} end

data/lib/crabfarm/global_state.rb

@@ -0,0 +1,22 @@
+module Crabfarm
+  module GlobalState
+
+    def inside_crawler_app?
+      defined? CF_PATH
+    end
+
+    def app_path
+      CF_PATH
+    end
+
+    def memento_path(_name)
+      File.join(app_path, 'spec/mementos', _name + '.json.gz')
+    end
+
+    def snapshot_path(_file)
+      File.join(app_path, 'spec/snapshots', _file)
+    end
+
+    extend self
+  end
+end

data/lib/crabfarm/modes/console.rb

@@ -1,28 +1,23 @@
-require 'benchmark'
 require 'readline'
 require 'rainbow'
 require 'rainbow/ext/string'
 require 'json'
+require 'crabfarm/engines/sync_state_loop'
 
 module Crabfarm
   module Modes
     class Console
 
-      class ConsoleDsl
-
-        def initialize(_context)
-          @context = _context
-        end
+      class ConsoleDsl < Engines::SyncStateManager
 
         def reload!
           puts "Reloading crawler source".color(:green)
-          ActiveSupport::Dependencies.clear
-          @context.reset
+          super
         end
 
         def reset
           puts "Resetting crawling context".color(:green)
-          @context.reset
+          super
         end
 
         def transition(_name=nil, _params={})
@@ -32,17 +27,14 @@ module Crabfarm
           end
 
           begin
-            elapsed = Benchmark.measure do
-              puts "Transitioning to #{_name.to_s.camelize} state"
-              doc = TransitionService.apply_state(@context, _name, _params).output_as_json
+            puts "Transitioning to #{_name.to_s.camelize} state"
+            output = super
 
-              puts "State changed, generated document:"
-              puts JSON.pretty_generate(doc).color(:green).gsub(/(^|\\n)/, '  ')
-            end
-            puts "Completed in #{elapsed.real} s"
-          rescue EntityNotFoundError => e
-            puts "#{e.to_s}".color(:red)
-          rescue => e
+            puts "State changed, generated document:"
+            puts JSON.pretty_generate(output.doc).color(:green).gsub(/(^|\\n)/, '  ')
+            puts "Completed in #{output.elapsed.real} s"
+
+          rescue Exception => e
             puts "#{e.to_s}".color(:red)
             puts e.backtrace
           end
@@ -56,7 +48,7 @@ module Crabfarm
         alias :r :reset
       end
 
-      def self.start(_context)
+      def self.process_input(_context)
         dsl = ConsoleDsl.new _context
 
         loop do
@@ -71,8 +63,7 @@ module Crabfarm
           end
         end
 
-        puts "Releasing crawling context".color(:green)
-        _context.release
+        puts "Exiting".color(:green)
       end
 
     end

data/lib/crabfarm/modes/generator.rb

@@ -9,7 +9,7 @@ module Crabfarm
     module Generator
 
       def generate_app(_target, _name, _default_remote=nil)
-        with_external_path _target do
+        with_base_path _target do
           binding = {
             name: _name,
             remote: _default_remote,
@@ -35,38 +35,29 @@ module Crabfarm
         end
       end
 
-      def generate_state(_name)
+      def generate_state(_target, _name)
         class_name = _name.camelize
-        with_crawler_path do
+        with_base_path _target do
           binding = { state_class: class_name.camelize }
           path('app', 'states', class_name.underscore + '.rb').render('state.rb', binding)
           path('spec', 'states', class_name.underscore + '_spec.rb').render('state_spec.rb', binding)
         end
       end
 
-      def generate_parser(_name)
+      def generate_parser(_target, _name)
         class_name = _name.camelize + 'Parser'
-        with_crawler_path do
+        with_base_path _target do
           binding = { parser_class: class_name }
           path('app', 'parsers', class_name.underscore + '.rb').render('parser.rb', binding)
           path('spec', 'parsers', class_name.underscore + '_spec.rb').render('parser_spec.rb', binding)
         end
       end
 
-      def with_external_path(_target)
+      def with_base_path(_target)
         @base_path = _target
         yield
       end
 
-      def with_crawler_path
-        if defined? CF_PATH
-          @base_path = CF_PATH
-          yield
-        else
-          puts "This command can only be run inside a crabfarm application"
-        end
-      end
-
       def path(*_args)
         @path = _args
         self

data/lib/crabfarm/modes/publisher.rb

@@ -14,7 +14,7 @@ module Crabfarm
     module Publisher
       extend self
 
-      DEFAULT_HOST = 'http://www.crabfarm.io'
+      DEFAULT_HOST = 'http://api.crabfarm.io'
 
       def publish(_path, _options={})
 

data/lib/crabfarm/modes/recorder.rb

@@ -4,37 +4,42 @@ require 'crabfarm/crabtrap_runner'
 
 module Crabfarm
   module Modes
-    class Recorder
+    module Recorder
 
-      def self.start(_target)
-        return puts "Must provide a recording name" unless _target.is_a? String
+      def self.start(_target, _replay=false)
+        return puts "Must provide a recording target" unless _target.is_a? String
+        return puts "Memento file does not exist: #{_target}" if _replay and not File.exist? _target
 
         crabtrap_config = Crabfarm.config.crabtrap_config
-        crabtrap_config[:mode] = :capture
-        crabtrap_config[:bucket_path] = File.join(CF_PATH, 'spec/mementos', _target + '.json.gz')
+        crabtrap_config[:mode] = _replay ? :replay : :capture
+        crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
+        crabtrap_config[:bucket_path] = _target
 
         crabtrap = CrabtrapRunner.new crabtrap_config
         crabtrap.start
 
-        driver_config = Crabfarm.config.driver_config
-        driver_config[:name] = :firefox
-        driver_config[:proxy] = "127.0.0.1:#{crabtrap.port}"
-
-        driver = DefaultDriverFactory.new(driver_config).build_driver nil
-
         begin
-          puts "Press Ctrl-C to stop capturing."
-          loop do
-            driver.current_url
-            sleep 1.0
+          driver_config = Crabfarm.config.driver_config
+          driver_config[:name] = Crabfarm.config.recorder_driver
+          driver_config[:proxy] = "127.0.0.1:#{crabtrap.port}"
+
+          driver = DefaultDriverFactory.new(driver_config).build_driver nil
+
+          begin
+            puts "Press Ctrl-C to stop capturing."
+            loop do
+              driver.current_url
+              sleep 1.0
+            end
+          rescue Selenium::WebDriver::Error::WebDriverError, SystemExit, Interrupt
+            # noop
           end
-        rescue Selenium::WebDriver::Error::WebDriverError, SystemExit, Interrupt
-          # noop
-        end
 
-        puts "Releasing crawling context".color(:green)
-        driver.quit rescue nil
-        crabtrap.stop
+          puts "Releasing crawling context".color(:green)
+          driver.quit rescue nil
+        ensure
+          crabtrap.stop
+        end
       end
 
     end

data/lib/crabfarm/modes/server.rb

@@ -1,6 +1,6 @@
 require 'grape'
 require 'crabfarm/support/custom_puma'
-require 'crabfarm/engines/safe_state_loop'
+require 'crabfarm/engines/async_state_manager'
 
 module Crabfarm
   module Modes
@@ -54,7 +54,7 @@ module Crabfarm
           optional :wait, type: Float
         end
         put :state do
-          print_state evaluator.change_state params[:name], params[:params], wait
+          print_state evaluator.transition params[:name], params[:params], wait
         end
       end
 
@@ -62,8 +62,8 @@ module Crabfarm
         @@evaluator
       end
 
-      def self.start(_options)
-        @@evaluator = Engines::SafeStateLoop.new
+      def self.serve(_context, _options)
+        @@evaluator = Engines::AsyncStateManager.new _context
         @@evaluator.start
         begin
           Support::CustomPuma.run API, _options

data/lib/crabfarm/phantom_runner.rb

@@ -5,16 +5,17 @@ module Crabfarm
 
     PHANTOM_START_TM = 5 # seconds
 
-    attr_reader :port
-
     def initialize(_config={})
       @config = _config;
       @pid = nil
     end
 
+    def port
+      @config[:port]
+    end
+
     def start
-      find_available_port
-      Crabfarm.logger.info "Starting phantomjs in port #{@port}"
+      Crabfarm.logger.info "Starting phantomjs in port #{port}"
       @pid = spawn_phantomjs
       Crabfarm.logger.info "Phantomjs started (PID: #{@pid})"
     end
@@ -32,22 +33,25 @@ module Crabfarm
   private
 
     def spawn_phantomjs
-      pid = Process.spawn({}, phantomjs_cmd)
+      pid = nil
       begin
+        pid = Process.spawn({}, phantomjs_cmd)
         Timeout::timeout(PHANTOM_START_TM) { wait_for_server }
+        return pid
+      rescue Errno::ENOENT
+        raise BinaryMissingError.new 'phantomjs', @config[:bin_path]
       rescue Timeout::Error
         Process.kill "INT", pid
         Process.wait pid
         raise
       end
-      return pid
     end
 
     def phantomjs_cmd
       cmd = [@config[:bin_path]]
       cmd << '--load-images=false' unless @config[:load_images]
       cmd << "--proxy=#{@config[:proxy]}" unless @config[:proxy].nil?
-      cmd << "--webdriver=#{@port}"
+      cmd << "--webdriver=#{port}"
       cmd << "--ssl-protocol=#{@config[:ssl]}" unless @config[:ssl].nil?
       cmd << "--ignore-ssl-errors=true"
       cmd << "--webdriver-loglevel=WARN"
@@ -55,37 +59,15 @@ module Crabfarm
       cmd.join(' ')
     end
 
-    def find_available_port
-      with_lock do
-        socket = Socket.new(:INET, :STREAM, 0)
-        socket.bind(Addrinfo.tcp("127.0.0.1", 0))
-        @port = socket.local_address.ip_port
-        socket.close
-      end
-    end
-
     def wait_for_server
       loop do
         begin
-          Net::HTTP.get_response(URI.parse("http://127.0.0.1:#{@port}/status"))
+          Net::HTTP.get_response(URI.parse("http://127.0.0.1:#{port}/status"))
           break
         rescue
         end
       end
     end
 
-    def with_lock
-      return yield if @config[:lock_file].nil?
-
-      File.open(@config[:lock_file], 'a+') do |file|
-        begin
-          file.flock File::LOCK_EX
-          return yield
-        ensure
-          file.flock File::LOCK_UN
-        end
-      end
-    end
-
   end
 end

data/lib/crabfarm/rspec.rb

@@ -1,26 +1,22 @@
-require 'crabfarm/crabtrap_context'
-require 'net/http'
-
-CF_TEST_CONTEXT = Crabfarm::CrabtrapContext::new
-CF_TEST_CONTEXT.load
-
 module Crabfarm
   module RSpec
 
-    def parse(_snap_or_url, _options={})
-      fixture = Pathname.new(File.join(ENV['SNAPSHOT_DIR'], _snap_or_url))
-      html = if fixture.exist?
-        File.read fixture.realpath
-      else
-        Net::HTTP.get(URI.parse _snap_or_url)
-      end
+    class Error < Crabfarm::Error; end
 
+    def parse(_snapshot, _options={})
+      snapshot_path = GlobalState.snapshot_path _snapshot
+      raise Error.new "Snapshot does not exist #{_snapshot}" unless File.exist? snapshot_path
+
+      html = File.read snapshot_path
       parser = described_class.new html, _options
       parser.parse
       parser
     end
 
     def crawl(_state=nil, _params={})
+
+      raise Error.new "Crawl is only available in state specs" if @context.nil?
+
       if _state.is_a? Hash
         _params = _state
         _state = nil
@@ -28,9 +24,9 @@ module Crabfarm
 
       if _state.nil?
         return nil unless described_class < BaseState # TODO: maybe raise an error here.
-        @state = @last_state = TransitionService.apply_state CF_TEST_CONTEXT, described_class, _params
+        @state = @last_state = TransitionService.apply_state @context, described_class, _params
       else
-        @last_state = TransitionService.apply_state CF_TEST_CONTEXT, _state, _params
+        @last_state = TransitionService.apply_state @context, _state, _params
       end
     end
 
@@ -46,26 +42,30 @@ module Crabfarm
       @parser
     end
 
+    def driver(_session_id=nil)
+      @context.pool.driver(_session_id)
+    end
+
   end
 end
 
 RSpec.configure do |config|
   config.include Crabfarm::RSpec
 
-  config.before(:example) do |example|
-
-    if example.metadata[:parsing]
-      @parser = parse example.metadata[:parsing], example.metadata[:using] || {}
-    end
-
-    if example.metadata[:crawling]
-      CF_TEST_CONTEXT.replay File.join(CF_PATH, 'spec/mementos', example.metadata[:crawling] + '.json.gz')
+  config.around(:example) do |example|
+    if described_class < Crabfarm::BaseParser
+      if example.metadata[:parsing]
+        @parser = parse example.metadata[:parsing], example.metadata[:using] || {}
+      end
+      example.run
+    elsif described_class < Crabfarm::BaseState
+      Crabfarm::ContextFactory.with_context example.metadata[:crawling] do |ctx|
+        @context = ctx
+        example.run
+      end
     else
-      CF_TEST_CONTEXT.pass_through
+      example.run
     end
   end
 
-  config.after(:suite) do
-    CF_TEST_CONTEXT.release
-  end
 end

data/lib/crabfarm/templates/Crabfile.erb

@@ -65,10 +65,13 @@ set_driver :phantomjs
 # set_driver_remote_timeout 120
 
 
-# Crabtrap launcher configuration
+# Recording configuration
 ########################################
 
-# The following parameters only apply when running the crabtrap proxy on specs
+# The following parameters only apply when recording or replaying mementos
 
 # Set the crabtrap executable location, by default crabfarm expects crabtrap to be included in enviroment
 # set_crabtrap_bin_path 'crabtrap'
+
+# Change the browser used for the recording command, available options are 'firefox' or 'chrome'
+# set_recorder_driver :firefox

data/lib/crabfarm/templates/spec_helper.rb.erb

@@ -1,8 +1,6 @@
 require File.expand_path("../../boot", __FILE__)
 Bundler.require :test
 
-ENV['SNAPSHOT_DIR'] ||= File.expand_path("../snapshots", __FILE__)
-
 require "crabfarm/rspec"
 
 RSpec.configure do |config|

data/lib/crabfarm/transition_service.rb

@@ -6,7 +6,7 @@ module Crabfarm
         load_by_name _name
       else _name end
 
-      _context.load
+      _context.prepare
       state = state_class.new _context, _params
       state.crawl
       state

data/lib/crabfarm/utils/port_discovery.rb

@@ -0,0 +1,17 @@
+module Crabfarm
+  module Utils
+    module PortDiscovery
+
+      def self.find_available_port
+        begin
+          socket = Socket.new(:INET, :STREAM, 0)
+          socket.bind(Addrinfo.tcp("127.0.0.1", 0))
+          return socket.local_address.ip_port
+        ensure
+          socket.close rescue nil
+        end
+      end
+
+    end
+  end
+end

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-24 00:00:00.000000000 Z
+date: 2015-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.33'
+        version: '2.45'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.33'
+        version: '2.45'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -377,6 +377,7 @@ files:
 - lib/crabfarm/cli.rb
 - lib/crabfarm/configuration.rb
 - lib/crabfarm/context.rb
+- lib/crabfarm/context_factory.rb
 - lib/crabfarm/crabtrap_context.rb
 - lib/crabfarm/crabtrap_runner.rb
 - lib/crabfarm/default_driver_factory.rb
@@ -385,10 +386,11 @@ files:
 - lib/crabfarm/dsl/surfer/search_context.rb
 - lib/crabfarm/dsl/surfer/surf_context.rb
 - lib/crabfarm/dsl/surfer.rb
-- lib/crabfarm/engines/safe_state_loop.rb
+- lib/crabfarm/engines/async_state_manager.rb
+- lib/crabfarm/engines/sync_state_manager.rb
 - lib/crabfarm/errors.rb
-- lib/crabfarm/event_store.rb
 - lib/crabfarm/forked_state.rb
+- lib/crabfarm/global_state.rb
 - lib/crabfarm/http_client.rb
 - lib/crabfarm/mocks/noop_driver.rb
 - lib/crabfarm/modes/console.rb
@@ -417,6 +419,7 @@ files:
 - lib/crabfarm/templates/state.rb.erb
 - lib/crabfarm/templates/state_spec.rb.erb
 - lib/crabfarm/transition_service.rb
+- lib/crabfarm/utils/port_discovery.rb
 - lib/crabfarm/version.rb
 - lib/crabfarm.rb
 - bin/crabfarm

data/lib/crabfarm/event_store.rb

@@ -1,20 +0,0 @@
-module Crabfarm
-  class EventStore
-
-    def initialize
-      @events = []
-      @mutex = Mutex.new
-    end
-
-    def event(_category, _message)
-      @mutex.synchronize do
-        @events << {
-          created_at: Time.current,
-          category: _category,
-          msg: _message
-        }
-      end
-    end
-
-  end
-end