crabfarm 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 09ceb80d5125962d7f54f0b7d9f79bc568221f18
-  data.tar.gz: f72e8f9951b3ae981bece342c9a7364b007677e9
+  metadata.gz: d58ed7568693ec48b019ca4bf3d8051073938d1b
+  data.tar.gz: c0853a7063ef07de79950d23802993c5d5f4bc27
 SHA512:
-  metadata.gz: 1572b106739d6f9a7b2ae385bb640b926f05707e31daf0a99758bb308ab8020a374636fc9df7eb56cf868c3a67aaa861d9538d859438367bc7a300e440613bf5
-  data.tar.gz: 39049c8c5b3b5c115dfdc11c92a67bb81411aff7512c16b524c6874c66cf0638adcb9a1a369628e83f5a50d667cabb8c494325f534b94a468ea941969f9790b7
+  metadata.gz: cd5f294f3424f16316633894cfa3e25bfa464ef3a56aaa099d572a1eaf74873de840e89ee7afd7ff7f047f261d7179a9fb9c85c6549ba69b09cb4228e0a93a8c
+  data.tar.gz: a27f2e4ff7adef226539f4ff24301d0c6c2162c47b4a35d519a1737aa0bc4d76878c1b5518cd4de1e5e47454393ba6ac08880364008bd1a620296e7c3c0e4763

data/lib/crabfarm/adapters/browser/phantom_js.rb

@@ -31,6 +31,7 @@ module Crabfarm
         def phantom_config
           {
             load_images: Crabfarm.config.phantom_load_images,
+            log_level: Crabfarm.config.phantom_log_level,
             ssl: Crabfarm.config.phantom_ssl,
             bin_path: Crabfarm.config.phantom_bin_path,
             proxy: config[:proxy],

data/lib/crabfarm/base_navigator.rb

@@ -22,7 +22,7 @@ module Crabfarm
     end
 
     def navigate(_name, _params={})
-      TransitionService.transition(@context, _name, params.merge(_params)).navigator
+      TransitionService.transition(@context, _name, params.merge(_params))
     end
 
     alias :nav :navigate

data/lib/crabfarm/base_reducer.rb

@@ -41,10 +41,6 @@ module Crabfarm
       raise NotImplementedError.new
     end
 
-    def as_json(_options=nil)
-      field_hash
-    end
-
     def to_json(_options=nil)
       field_hash.to_json _options
     end

data/lib/crabfarm/base_struct.rb

@@ -11,10 +11,6 @@ module Crabfarm
       _values.each { |k,v| send("#{k}=", v) }
     end
 
-    def as_json(_options=nil)
-      field_hash
-    end
-
     def to_json(_options={})
       field_hash.to_json(_options)
     end

data/lib/crabfarm/configuration.rb

@@ -22,17 +22,16 @@ module Crabfarm
 
       # Phantom launcher configuration
       [:phantom_load_images, :boolean, 'Phantomjs image loading, only for phantomjs driver.'],
+      [:phantom_log_level, ['error', 'warn', 'info', 'debug'], 'Phantomjs log level.'],
       [:phantom_ssl, ['sslv3', 'sslv2', 'tlsv1', 'any'], 'Phantomjs ssl mode: sslv3, sslv2, tlsv1 or any, only for phantomjs driver.'],
       [:phantom_bin_path, :string, 'Phantomjs binary path, only for phantomjs driver.'],
 
       # Crabtrap launcher configuration
       [:crabtrap_bin_path, :string, 'Crabtrap binary path.'],
+      [:crabtrap_log_level, ['error', 'warn', 'info', 'debug'], 'Crabtrap log level'],
 
       # Recorder configuration
-      [:recorder_driver, :string, 'Recorder driver name, defaults to \'firefox\''],
-
-      # Live configurations
-      [:live_full_reload, :boolean, 'Whether to create a new driver session on every live reload or not']
+      [:recorder_driver, :string, 'Recorder driver name, defaults to \'firefox\'']
     ]
     .map { |o| Option.new(*o) }
 
@@ -67,9 +66,10 @@ module Crabfarm
         phantom_load_images: false,
         phantom_ssl: 'any',
         phantom_bin_path: 'phantomjs',
+        phantom_log_level: :warn,
         crabtrap_bin_path: 'crabtrap',
-        recorder_driver: :firefox,
-        live_full_reload: false
+        crabtrap_log_level: :warn,
+        recorder_driver: :firefox
       }
     end
 
@@ -87,6 +87,7 @@ module Crabfarm
     def crabtrap_config
       {
         bin_path: crabtrap_bin_path,
+        log_level: crabtrap_log_level,
         proxy: proxy
       }
     end

data/lib/crabfarm/crabtrap_runner.rb

@@ -69,6 +69,7 @@ module Crabfarm
       cmd << @config[:bucket_path] if mode != :pass
       cmd << "--port=#{port}"
       cmd << "--virtual=#{@config[:virtual]}" if @config.key? :virtual
+      cmd << "--log=#{@config[:log_level].to_s.upcase}"
       cmd
     end
 

data/lib/crabfarm/engines/async_state_manager.rb

@@ -1,4 +1,3 @@
-require 'benchmark'
 require 'ostruct'
 
 module Crabfarm
@@ -106,18 +105,19 @@ module Crabfarm
             if @working
               begin
                 logger.info "Transitioning state: #{@next_state_name}"
-                @elapsed = Benchmark.measure do
-                  ActiveSupport::Dependencies.clear
-                  @doc = TransitionService.transition(@context, @next_state_name, @next_state_params).document
-                end.real
+                ActiveSupport::Dependencies.clear
+                ts = TransitionService.transition(@context, @next_state_name, @next_state_params)
+                @elapsed = ts.elapsed
+                @doc = ts.document
+                @error = nil
 
                 logger.info "Transitioned in #{@elapsed.real}"
-                @error = nil
               rescue Exception => e
-                logger.error "Error during transition:"
-                logger.error e
                 @doc = nil
                 @error = e
+
+                logger.error "Error during transition:"
+                logger.error e
               end
 
               @lock.synchronize {

data/lib/crabfarm/engines/sync_state_manager.rb

@@ -1,4 +1,3 @@
-require 'benchmark'
 require 'ostruct'
 
 module Crabfarm
@@ -27,13 +26,14 @@ module Crabfarm
 
       def navigate(_name, _params={})
         @lock.synchronize {
-          output = { name: _name, params: _params }
-
-          output[:elapsed] = Benchmark.measure do
-            output[:doc] = TransitionService.transition(@context, _name, _params).document
-          end
-
-          OpenStruct.new output
+          ts = TransitionService.transition(@context, _name, _params)
+
+          OpenStruct.new({
+            name: _name,
+            params: _params,
+            doc: ts.document,
+            elapsed: ts.elapsed
+          })
         }
       end
 

data/lib/crabfarm/live/navigator_runner_direct.rb

@@ -1,4 +1,3 @@
-require 'benchmark'
 require 'crabfarm/utils/console'
 
 module Crabfarm
@@ -14,9 +13,7 @@ module Crabfarm
 
       def execute
         Crabfarm.with_context @memento do |ctx|
-          @elapsed = Benchmark.measure do
-            @transition = TransitionService.transition ctx, @target, (@params || {})
-          end
+          @transition = TransitionService.transition ctx, @target, (@params || {})
         end
       end
 
@@ -24,7 +21,7 @@ module Crabfarm
         @manager.show_message(
           :neutral,
           'Navigation completed!',
-          "The page was scrapped in #{@elapsed.real} seconds",
+          "The page was scrapped in #{@transition.elapsed} seconds",
           @transition.document.to_json,
           :json
         )

data/lib/crabfarm/modes/console.rb

@@ -2,6 +2,7 @@ require 'readline'
 require 'rainbow'
 require 'rainbow/ext/string'
 require 'json'
+require 'crabfarm/utils/console'
 require 'crabfarm/engines/sync_state_manager'
 
 module Crabfarm
@@ -23,60 +24,62 @@ module Crabfarm
         end
 
         def reload!
-          puts "Reloading crawler source".color Colors::NOTICE
+          console.info "Reloading crawler source"
           @manager.reload
           nil
         end
 
         def reset
-          puts "Resetting crawling context".color Colors::NOTICE
+          console.info "Resetting crawling context"
           @manager.reset
           nil
         end
 
         def navigate(_name=nil, _params={})
           if _name.nil?
-            puts "Must provide a navigator name".color Colors::ERROR
+            console.warning "Must provide a navigator name"
             return
           end
 
           begin
-            puts "Navigating...".color Colors::NOTICE
+            console.info "Navigating..."
             output = @manager.navigate _name, _params
-            puts JSON.pretty_generate(output.doc).gsub(/(^|\\n)/, '  ').color Colors::RESULT
-            puts "Completed in #{output.elapsed.real} s".color Colors::NOTICE
+            console.json_result output.doc
+            console.info "Completed in #{output.elapsed.real} s"
 
-          rescue Exception => e
-            puts "#{e.to_s}".color Colors::ERROR
-            puts e.backtrace
+          rescue Exception => exc
+            console.exception exc
           end
         end
 
         def snap(_name=nil, _params={})
           if _name.nil?
-            puts "Must provide a navigator name".color Colors::ERROR
+            console.warning "Must provide a navigator name"
             return
           end
 
           begin
-            puts "Navigating, waiting to hit a reducer...".color Colors::NOTICE
+            console.info "Navigating, waiting to hit a reducer..."
             require 'crabfarm/modes/shared/snapshot_decorator'
             Factories::Reducer.with_decorator Shared::SnapshotDecorator do
               @manager.navigate _name, _params
             end
-            puts "Navigation completed".color Colors::NOTICE
+            console.info "Navigation completed"
 
-          rescue Exception => e
-            puts "#{e.to_s}".color Colors::ERROR
-            puts e.backtrace
+          rescue Exception => exc
+            console.exception exc
           end
         end
 
         def help
-          puts "Ejem..."
+          console.info "Ejem..."
           nil
         end
 
+        def console
+          Crabfarm::Utils::Console
+        end
+
         alias :nav :navigate
       end
 
@@ -88,16 +91,15 @@ module Crabfarm
             output = dsl.instance_eval Readline.readline("> ", true)
             puts output.inspect unless output.nil?
           rescue SyntaxError => se
-            puts "Syntax error: #{se.message}".color(Colors::ERROR)
+            Crabfarm::Utils::Console.exception se
           rescue SystemExit, Interrupt
             break
-          rescue => e
-            puts "#{e.to_s}".color(Colors::ERROR)
-            puts e.backtrace
+          rescue => exc
+            Crabfarm::Utils::Console.exception exc
           end
         end
 
-        puts "Exiting".color(Colors::NOTICE)
+        Crabfarm::Utils::Console.system "Exiting"
       end
 
     end

data/lib/crabfarm/modes/server.rb

@@ -14,7 +14,7 @@ module Crabfarm
         prefix :api
 
         rescue_from Grape::Exceptions::ValidationErrors do |e|
-          rack_response({ errors: e.as_json }.to_json, 400)
+          rack_response({ errors: e }.to_json, 400)
         end
 
         rescue_from Crabfarm::ApiError do |e|
@@ -54,7 +54,7 @@ module Crabfarm
           optional :wait, type: Float
         end
         put :state do
-          print_state evaluator.navigate params[:name], params[:params], wait
+          print_state evaluator.navigate params[:name], params[:params] || {}, wait
         end
       end
 

data/lib/crabfarm/rspec.rb

@@ -14,7 +14,7 @@ RSpec.configure do |config|
         example.run
       ensure
         # store result in metadata so it can be accessed by formatters/reporters
-        example.metadata[:result] = @reducer_state.as_json if @reducer_state
+        example.metadata[:result] = @reducer_state if @reducer_state
       end
     elsif described_class < Crabfarm::BaseNavigator
       Crabfarm.with_context example.metadata[:navigating] do |ctx|

data/lib/crabfarm/support/phantom_runner.rb

@@ -55,7 +55,7 @@ module Crabfarm
         cmd << "--ssl-protocol=#{@config[:ssl]}" unless @config[:ssl].nil?
         cmd << "--ignore-ssl-errors=true"
         cmd << "--web-security=false"
-        cmd << "--webdriver-loglevel=WARN"
+        cmd << "--webdriver-loglevel=#{@config[:log_level].to_s.upcase}"
         cmd
       end
 

data/lib/crabfarm/templates/Crabfile.erb

@@ -36,7 +36,10 @@ set_webdriver_dsl :pincers
 # The following parameters only apply if using the :phantomjs webdriver
 
 # Make phantom load images when requesting documents
-# set_phantom_load_images false
+# set_phantom_load_images true
+
+# Set phantom log level, defaults to warning
+# set_phantom_log_level :warn
 
 # Set phantom ssl mode, defaults to :any, posible options are sslv3, sslv2, tlsv1 or any
 # set_phantom_ssl 'any'
@@ -70,10 +73,3 @@ set_webdriver_dsl :pincers
 
 # Change the browser used for the recording command, available options are 'firefox' or 'chrome'
 # set_recorder_driver :firefox
-
-
-# Live mode configuration
-########################################
-
-# Build a new webdriver session on each reload
-# set_live_full_reload false

data/lib/crabfarm/templates/Gemfile.erb

@@ -1,7 +1,7 @@
 source 'https://rubygems.org'
 
 gem "crabfarm", '<%= version %>'
-gem "pincers", '~> 0.2.1'
+gem "pincers", '~> 0.6'
 
 # Comment this if not using a nokogiri based parser or browser
 gem 'nokogiri', "~> 1.6.6"

data/lib/crabfarm/transition_service.rb

@@ -1,3 +1,5 @@
+require 'benchmark'
+
 module Crabfarm
   class TransitionService
 
@@ -5,7 +7,7 @@ module Crabfarm
       self.new(_context).transition(_name, _params)
     end
 
-    attr_reader :document, :navigator
+    attr_reader :document, :navigator, :elapsed
 
     def initialize(_context)
       @context = _context
@@ -14,10 +16,11 @@ module Crabfarm
     def transition(_name_or_class, _params={})
       navigator_class = Utils::Resolve.navigator_class _name_or_class
 
-      @context.prepare
-      @navigator = Factories::Navigator.build navigator_class, @context, _params
-      @document = @navigator.run
-      @document = @document.as_json if @document.respond_to? :as_json
+      @elapsed = Benchmark.measure do
+        @context.prepare
+        @navigator = Factories::Navigator.build navigator_class, @context, _params
+        @document = @navigator.run
+      end.real
 
       self
     end

data/lib/crabfarm/utils/processes.rb

@@ -1,30 +1,33 @@
 require 'childprocess'
+require 'crabfarm/utils/stream_dumper'
 
 ChildProcess.posix_spawn = true
 
-module Crabfarm
-  module Utils
-    module Processes
+module Crabfarm::Utils
 
-      def self.start_logged_process(_name, _cmd, _logger, _env={})
-        ro, wo = IO.pipe
-        re, we = IO.pipe
+  module Processes
+    def self.start_logged_process(_name, _cmd, _logger, _env={})
 
-        proc = ChildProcess.build(*_cmd)
-        proc.environment.merge! _env
-        proc.io.stdout = wo
-        proc.io.stderr = we
-        proc.start
+      ro, wo = IO.pipe
+      re, we = IO.pipe
 
-        wo.close
-        we.close
+      proc = ChildProcess.build(*_cmd)
+      proc.environment.merge! _env
+      proc.io.stdout = wo
+      proc.io.stderr = we
+      proc.start
 
-        Thread.new { ro.each_line { |l| _logger.info "[#{_name.upcase}] #{l}" } }
-        Thread.new { re.each_line { |l| _logger.warn "[#{_name.upcase}] #{l}" } }
+      # close write endpoints after fork
+      wo.close
+      we.close
 
-        proc
-      end
+      # register log consumers
+      StreamDumper.register_stream(ro) { |l| _logger.info "[#{_name.upcase}] #{l}" }
+      StreamDumper.register_stream(re) { |l| _logger.warn "[#{_name.upcase}] #{l}" }
 
+      proc
     end
   end
+
 end
+

data/lib/crabfarm/utils/stream_dumper.rb

@@ -0,0 +1,142 @@
+require 'thwait'
+
+module Crabfarm::Utils
+
+  class StreamDumper
+
+    class Reload < StandardError; end
+
+    def self.consumer
+      @@consumer ||= self.new
+    end
+
+    def self.register_stream(_stream, &_block)
+      consumer.register_stream _stream, &_block
+      consumer.restart
+    end
+
+    def initialize
+      @streams = []
+      @lock = Mutex.new
+      @worker = nil
+    end
+
+    def register_stream(_stream, &_block)
+      @lock.synchronize { @streams << Wrapper.new(_stream, _block) }
+    end
+
+    def restart
+      @lock.synchronize do
+        if @worker and @worker.alive?
+          @worker.raise Reload # signal worker to reload streams
+        else
+          @worker = load_worker
+        end
+      end
+    end
+
+  private
+
+    def load_worker
+      Thread.new do
+        looped = 0
+        begin
+          while @streams.count > 0
+            all_streams = dump_streams
+            result = IO.select(all_streams << $stdin, [], all_streams)
+            if result
+              process_ready_streams result[0]
+              discard_errored_streams result[2]
+            end
+            looped += 1
+          end
+        rescue Reload
+          retry
+        rescue SystemExit
+          # nothing
+        end
+      end
+    end
+
+    def process_ready_streams(_streams)
+      _streams.each do |stream|
+        wrapper = find_stream_wrapper stream
+        if wrapper
+          wrapper.process_lines
+          remove_wrapper wrapper, :eof if wrapper.eof?
+        end
+      end
+    end
+
+    def discard_errored_streams(_streams)
+      _streams.each do |stream|
+        wrapper = find_stream_wrapper stream
+        remove_wrapper wrapper, :error if wrapper
+      end
+    end
+
+    def find_stream_wrapper(_stream)
+      @streams.find { |s| s.stream == _stream }
+    end
+
+    def remove_wrapper(_wrapper, _reason)
+      @lock.synchronize { @streams.delete _wrapper }
+    end
+
+    def dump_streams
+      @streams.map(&:stream)
+    end
+
+    class Wrapper
+
+      MAX_LENGTH = 2048
+
+      attr_reader :stream
+
+      def initialize(_stream, _block)
+        @stream = _stream
+        @block = _block
+        @buffer = []
+        @eof = false
+      end
+
+      def eof?
+        @eof
+      end
+
+      def process_lines
+        begin
+          loop do
+            chunk = @stream.read_nonblock(MAX_LENGTH)
+            process_chunk chunk
+          end
+        rescue IO::WaitReadable
+          # nothing, just stop looping
+        rescue EOFError
+          @eof = true
+        rescue Exception => exc
+          puts "Error in stream consumer: #{exc}"
+          @eof = true
+        end
+      end
+
+      def process_chunk(_chunk)
+        index = _chunk.index $/
+
+        unless index.nil?
+          head = _chunk[0..index-1]
+          tail = _chunk[index+1..-1]
+
+          @block.call(@buffer.join + head)
+          @buffer.clear
+
+          process_chunk tail
+        else
+          @buffer << _chunk
+        end
+      end
+
+    end
+
+  end
+end

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.6.2"
+  VERSION = "0.7.0"
 end

data/lib/crabfarm.rb

@@ -50,9 +50,13 @@ module Crabfarm
 
   def self.logger
     if @@logger.nil?
-      @@logger = Logger.new(@@config.log_path.nil? ? STDOUT : File.join(@@config.log_path, 'crawler.log'))
-      @@logger.level = Logger::INFO
+      if @@config.log_path.nil? or @@config.log_path.empty?
+        @@logger = Logger.new STDOUT
+      else
+        @@logger = Logger.new File.join(@@config.log_path, 'crawler.log')
+      end
     end
+
     @@logger
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-20 00:00:00.000000000 Z
+date: 2015-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '3.0'
     - - <
       - !ruby/object:Gem::Version
         version: '5'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '3.0'
     - - <
       - !ruby/object:Gem::Version
         version: '5'
@@ -36,61 +36,64 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: '2.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: '2.12'
 - !ruby/object:Gem::Dependency
   name: inquirer
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: rainbow
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: grape
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.10'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.10.2
   type: :runtime
@@ -98,27 +101,33 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.10'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.10.2
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.12'
 - !ruby/object:Gem::Dependency
   name: childprocess
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.5.5
   type: :runtime
@@ -126,6 +135,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.5.5
 - !ruby/object:Gem::Dependency
@@ -146,16 +158,16 @@ dependencies:
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
@@ -175,6 +187,9 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.6.6
   type: :development
@@ -182,6 +197,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.6.6
 - !ruby/object:Gem::Dependency
@@ -190,14 +208,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: '0.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -216,77 +234,80 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.4'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: rspec-nc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.11'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.5'
 - !ruby/object:Gem::Dependency
   name: terminal-notifier-guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.6.1
   type: :development
@@ -294,55 +315,61 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.6.1
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: pry-remote
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: pry-nav
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.3.3
   type: :development
@@ -351,54 +378,46 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.3
-- !ruby/object:Gem::Dependency
-  name: capybara
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+        version: '1.3'
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.3
 - !ruby/object:Gem::Dependency
   name: watir-webdriver
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: jbuilder
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.6.7
   type: :development
@@ -406,9 +425,15 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.6.7
-description: 
+description: "\n    Crabfarm lets you build web scrappers using TDD, it is also very
+  opinionated about\n    the way a scrapper should be structured. Scrappers created
+  by crabfarm can be hosted\n    and monitored in the Crabfarm grid infrastructure.\n
+  \ "
 email:
 - ignacio@platan.us
 executables:
@@ -416,6 +441,14 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- assets/live-tools/instructions.html
+- assets/live-tools/selectorgadget_combined.css
+- assets/live-tools/selectorgadget_combined.js
+- assets/live-tools/tools.css
+- assets/live-tools/tools.js
+- assets/live-tools/welcome.html
+- bin/crabfarm
+- lib/crabfarm.rb
 - lib/crabfarm/adapters/browser/abstract_webdriver.rb
 - lib/crabfarm/adapters/browser/base.rb
 - lib/crabfarm/adapters/browser/chrome.rb
@@ -475,25 +508,25 @@ files:
 - lib/crabfarm/modes/server.rb
 - lib/crabfarm/modes/shared/interactive_decorator.rb
 - lib/crabfarm/modes/shared/snapshot_decorator.rb
+- lib/crabfarm/rspec.rb
 - lib/crabfarm/rspec/navigator_spec_helpers.rb
 - lib/crabfarm/rspec/reducer_spec_helpers.rb
 - lib/crabfarm/rspec/reducer_spy.rb
 - lib/crabfarm/rspec/reducer_spy_manager.rb
-- lib/crabfarm/rspec.rb
 - lib/crabfarm/state_store.rb
 - lib/crabfarm/strategies.rb
 - lib/crabfarm/support/custom_puma.rb
 - lib/crabfarm/support/gli.rb
 - lib/crabfarm/support/phantom_runner.rb
 - lib/crabfarm/support/webdriver_factory.rb
+- lib/crabfarm/templates/Crabfile.erb
+- lib/crabfarm/templates/Gemfile.erb
 - lib/crabfarm/templates/boot.rb.erb
 - lib/crabfarm/templates/crabfarm_bin.erb
-- lib/crabfarm/templates/Crabfile.erb
 - lib/crabfarm/templates/dot_crabfarm.erb
 - lib/crabfarm/templates/dot_gitignore.erb
 - lib/crabfarm/templates/dot_gitkeep.erb
 - lib/crabfarm/templates/dot_rspec.erb
-- lib/crabfarm/templates/Gemfile.erb
 - lib/crabfarm/templates/navigator.rb.erb
 - lib/crabfarm/templates/navigator_spec.rb.erb
 - lib/crabfarm/templates/reducer.rb.erb
@@ -507,16 +540,9 @@ files:
 - lib/crabfarm/utils/processes.rb
 - lib/crabfarm/utils/resolve.rb
 - lib/crabfarm/utils/rspec_runner.rb
+- lib/crabfarm/utils/stream_dumper.rb
 - lib/crabfarm/utils/webdriver.rb
 - lib/crabfarm/version.rb
-- lib/crabfarm.rb
-- bin/crabfarm
-- assets/live-tools/instructions.html
-- assets/live-tools/selectorgadget_combined.css
-- assets/live-tools/selectorgadget_combined.js
-- assets/live-tools/tools.css
-- assets/live-tools/tools.js
-- assets/live-tools/welcome.html
 homepage: https://github.com/platanus/crabfarm-gem
 licenses:
 - MIT
@@ -537,9 +563,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
-summary: Crabfarm crawler creation framework
+summary: Crabfarm is a TDD oriented web scrapping framework
 test_files: []
 has_rdoc: