micro_spider 0.1.23 → 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cedb84b01c8f87814486e68113ff9d69808749a0
+  data.tar.gz: 82fe718a38b980c0879b8b4eaec807b13b36e4fe
+SHA512:
+  metadata.gz: 28035e8b24d4cb6ac59fcf97fbf34f661c610e054de37c28b2ae7de4e79f002cebfddfb0fdc064e6f8c3475b68683fd1b07055add0d76c76e3ea5a4700f31697
+  data.tar.gz: d6924f477e659bf14954eb4893d0c9247b892920cd1583a37024eceae2a105130b2c3c98013f7069ede665df27e92f0458c71bdae33d5e182a4cb80b4b5acaf9

data/README.md

@@ -11,22 +11,15 @@ spider = MicroSpider.new
 spider.learn do
   site 'http://www.bbc.com'
   entrance '/news'
-  
-  field :top_story, '#top-story h2 a'
-  
-  follow '.story' do
-    
-    field :title, 'h1.story-header'
-    field :body,  '.story-body'
-
-    fields :related_stories, '.related-links-list a'
-    
-  end
-  
+  fields :top_stories, 'a.title-link'
 end
 
 spider.crawl
 
+spider.get('top_stories')
+# or
+spider.excretion['/news']['top_stories']
+
 ```
 
 

data/lib/micro_spider.rb

@@ -1,3 +1,4 @@
+require 'hashie'
 require 'capybara'
 require 'capybara/dsl'
 require 'capybara/mechanize'
@@ -32,14 +33,16 @@ class MicroSpider
   include SpiderCore::PaginationDSL
 
   attr_reader   :excretion, :paths, :delay, :current_location, :visited_paths, :broken_paths
-  attr_accessor :logger, :actions, :recipe, :skip_set_entrance, :timeout
+  attr_accessor :logger, :actions, :recipe, :skip_set_entrance, :timeout, :selector
 
-  def initialize(excretion = nil)
+  def initialize(excretion = nil, selector: :css)
+    @selector         = selector
     @paths            = []
     @actions          = []
     @setted_variables = {}
     @timeout          = 120
-    @excretion        = excretion || { status: 'inprogress', results: [] }
+    @status           = 'pending'
+    @excretion        = excretion || SpiderCore::Excretion.new
     @logger           = Logger.new(STDOUT)
     @visited_paths    = Set.new
     @broken_paths     = []
@@ -67,7 +70,7 @@ class MicroSpider
     sleep_or_not
     logger.info "Begin to visit #{path}."
     super(path)
-    @current_location = {entrance: path}
+    @current_location = SpiderCore::Excretion['_path' => path]
     logger.info "Current location is #{path}."
   end
 
@@ -84,16 +87,15 @@ class MicroSpider
   #   spider.set :table, '.tb a', selector: :css do |e|
   #     e['src']
   #   end
-  def set(name, value, opts = {}, &block)
-    selector = opts.delete(:selector)
-    if selector.nil?
-      @setted_variables[name.to_s] = value
-    else
-      actions << lambda {
-        elements = scan_all(selector, value, opts)
-        @setted_variables[name.to_s] = block_given? ? yield(elements) : handle_element(elements.first)
-      }
-    end
+  def set(name, value)
+    @setted_variables[name.to_s] = value
+  end
+
+  def set_on(name, pattern, &block)
+    actions << lambda {
+      element = scan_first(pattern)
+      @setted_variables[name.to_s] = block_given? ? yield(element) : handle_element(element)
+    }
   end
 
   # Click the locator. This will trigger visit action and change current location.
@@ -107,8 +109,9 @@ class MicroSpider
         spider = self.spawn
         spider.entrance(path)
         spider.learn(&block)
-        current_location[:click] ||= []
-        current_location[:click] << spider.crawl[:results]
+        put(
+          "click::#{path}", spider.crawl
+        )
       else
         visit(path)
       end
@@ -153,7 +156,7 @@ class MicroSpider
 
   def site(url)
     return if @site
-    Capybara.app_host = @excretion[:site] = @site = url
+    Capybara.app_host = @site = url
   end
 
   # This will be the first path for spider to visit.
@@ -171,21 +174,25 @@ class MicroSpider
     @paths += path_or_paths
   end
 
+  def with(pattern, path:, &block)
+    visit(path)
+    scan_all(pattern).map{ |element| yield(element) }
+  end
+
   # Sometimes the entrances are on the page.
   # @param path [String] path to visit
   # @param pattern [String, Regexp] links pattern
   #
   # @example
   #   spider = MicroSpider.new
-  #   spider.entrance_on_path('http://google.com', '.links a')
+  #   spider.entrance_on('.links a')
+  #   spider.entrance_on('.links a', path: '/a')
   #
-  def entrance_on_path(path, pattern, opts = {}, &block)
+  def entrance_on(pattern, path: '/', attr: :href)
     return if @skip_set_entrance
-    kind = opts[:kind] || :css
+
     visit(path)
-    entrances = scan_all(kind, pattern, opts).map do |element|
-      block_given? ? yield(element) : element[:href]
-    end
+    entrances = scan_all(pattern).map{ |element| element[attr] }
     @paths += entrances.to_a
   end
 
@@ -209,12 +216,14 @@ class MicroSpider
 
     begin
       visit(path)
+      @status = 'inprogress'
     rescue Timeout::Error => err
       @broken_paths << path
       logger.fatal("Timeout!!! execution expired when visit `#{path}`")
       logger.fatal(err)
     rescue SystemExit, Interrupt
       logger.fatal("SystemExit && Interrupt")
+      @status = 'exit'
       exit!
     rescue Exception => err
       @broken_paths << path
@@ -224,8 +233,8 @@ class MicroSpider
     else
       @visited_paths << path
       execute_actions
-      yield(@current_location) if block_given?
-      excretion[:results] << @current_location
+      #yield(@current_location) if block_given?
+      @excretion = @excretion.put(path, @current_location)
     ensure
       @actions = []
       @skip_set_entrance = true
@@ -238,7 +247,8 @@ class MicroSpider
   def reset
     return unless completed?
     @paths            = visited_paths.to_a
-    @excretion        = { status: 'inprogress', results: [] }
+    @status           = 'pending'
+    @excretion        = nil
     @visited_paths    = Set.new
     @current_location = nil
   end
@@ -257,7 +267,7 @@ class MicroSpider
   #   spider.save
   #
   def create_action(name, &block)
-    action = proc { actions << lambda { block.call(current_location) } }
+    action = proc { actions << lambda { block.call(@excretion) } }
     metaclass.send :define_method, name, &action
   end
 
@@ -269,10 +279,12 @@ class MicroSpider
         logger.fatal('Timeout!!! execution expired when execute action')
         logger.fatal(err.message)
         logger.fatal(err.backtrace.inspect)
+        @visited_paths.pop
         break
       rescue SpiderCore::ClickPathNotFound => err
         logger.fatal(err.message)
         logger.fatal(err.backtrace.inspect)
+        @visited_paths.pop
         break
       end
     }
@@ -292,22 +304,26 @@ class MicroSpider
     spider         = self.class.new
     spider.logger  = logger
     spider.timeout = timeout
+    spider.site(@site)
     spider.learn(&block) if block_given?
     spider
   end
 
-  def results
-    excretion[:results]
-  end
-
   def completed?
-    excretion[:status] == 'completed'
+    @status == 'completed'
   end
 
   def metaclass
     class << self; self; end
   end
 
+  def get(field)
+    @_deep_fetch ||= excretion.extend Hashie::Extensions::DeepFind
+    result = @_deep_fetch.deep_find_all(field.to_s)
+    return if result.nil?
+    result.length == 1 ? result.pop : result
+  end
+
   # The default page is Capybara.current_session.
   # Share one page may cause difficult issue, so here i separate it.
   def page
@@ -335,7 +351,7 @@ class MicroSpider
     end
 
     def complete
-      excretion[:status] = 'completed'
+      @status = 'completed'
       suicide
     end
 

data/lib/spider_core.rb

@@ -3,3 +3,4 @@ require 'spider_core/follow_dsl'
 require 'spider_core/field_dsl'
 require 'spider_core/pagination_dsl'
 require 'spider_core/exceptions'
+require 'spider_core/excretion'

data/lib/spider_core/behavior.rb

@@ -1,14 +1,12 @@
-require 'enumerable/lazy' if RUBY_VERSION < '2.0'
-
 module SpiderCore
   module Behavior
 
     protected
 
-    def scan_all(kind, pattern, opts = {})
+    def scan_all(pattern, opts = {})
       pattern = handle_pattern(pattern)
       if pattern.is_a?(String)
-        elements = all(kind, pattern).lazy
+        elements = all(selector, pattern).lazy
         if opts[:limit] && opts[:limit].to_i > 0
           elements = elements.take(opts[:limit].to_i)
         end
@@ -18,10 +16,10 @@ module SpiderCore
       end
     end
 
-    def scan_first(kind, pattern)
+    def scan_first(pattern)
       pattern = handle_pattern(pattern)
       if pattern.is_a?(String)
-        first(kind, pattern)
+        first(selector, pattern)
       elsif pattern.is_a?(Regexp)
         html[pattern, 1]
       end
@@ -61,5 +59,9 @@ module SpiderCore
       pattern
     end
 
+    def put(display, value)
+      @current_location = @current_location.put(display, value)
+    end
+
   end
 end

data/lib/spider_core/excretion.rb

@@ -0,0 +1,13 @@
+require "hamster/hash"
+
+module SpiderCore
+  class Excretion < Hamster::Hash
+    def is_a?(t)
+      if t == ::Hash
+        true
+      else
+        super(t)
+      end
+    end
+  end
+end

data/lib/spider_core/field_dsl.rb

@@ -5,40 +5,22 @@ module SpiderCore
     #
     # @param display [String] display name
     def field(display, pattern, opts = {}, &block)
-      kind = opts[:kind] || :css
       actions << lambda {
-        action_for(:field, {display: display, pattern: pattern, kind: kind}, opts, &block)
+        action_for(:field, {display: display, pattern: pattern}, opts, &block)
       }
     end
 
-    def css_field(display, pattern, opts = {}, &block)
-      field(display, pattern, opts.merge(kind: :css), &block)
-    end
-
-    def xpath_field(display, pattern, opts = {}, &block)
-      field(display, pattern, opts.merge(kind: :xpath), &block)
-    end
-
     def fields(display, pattern, opts = {}, &block)
-      kind = opts[:kind] || :css
       actions << lambda {
-        action_for(:fields, {display: display, pattern: pattern, kind: kind}, opts, &block)
+        action_for(:fields, {display: display, pattern: pattern}, opts, &block)
       }
     end
 
-    def css_fields(display, pattern, opts = {}, &block)
-      fields(display, pattern, opts.merge(kind: :css), &block)
-    end
-
-    def xpath_fields(display, pattern, opts = {}, &block)
-      fields(display, pattern, opts.merge(kind: :xpath), &block)
-    end
-
     def foreach(pattern, opts = {}, &block)
       return unless block_given?
-      kind     = opts[:kind] || :css
+
       actions << lambda {
-        scan_all(kind, pattern).each do |element|
+        scan_all(pattern, opts).each do |element|
           yield(element)
         end
       }
@@ -52,24 +34,22 @@ module SpiderCore
 
         elements = case action
         when :field
-          scan_first(action_opts[:kind], action_opts[:pattern])
+          scan_first action_opts[:pattern]
         when :fields
-          scan_all(action_opts[:kind], action_opts[:pattern], opts)
+          scan_all action_opts[:pattern], opts
         else
           raise 'Unknow action.'
         end
 
-        make_field_result( action_opts[:display], handle_elements(elements, &block) )
+        put(
+          action_opts[:display].to_s,
+          handle_elements(elements, &block)
+        )
       rescue Exception => err
         logger.fatal("Caught exception when get `#{action_opts[:pattern]}`.")
         logger.fatal(err)
       end
     end
 
-    def make_field_result(display, field)
-      current_location[:field] ||= []
-      current_location[:field] << {display => field}
-    end
-
   end
 end

data/lib/spider_core/follow_dsl.rb

@@ -3,19 +3,21 @@ module SpiderCore
 
     attr_accessor :skip_followers
 
-    def follow(pattern, opts = {}, &block)
+    def follow(pattern, attr: :href, **opts, &block)
       return unless block_given?
-      kind = opts[:kind] || :css
+
       actions << lambda {
         spider = self.spawn
         spider.learn(&block)
-        scan_all(kind, pattern, opts).each do |element|
+        scan_all(pattern, opts).each do |element|
           next if skip_followers && skip_followers.include?(element[:href])
+
           spider.skip_set_entrance = false
-          spider.entrance(element[:href])
+          spider.entrance(element[attr])
         end
-        current_location[:follow] ||= []
-        current_location[:follow] << spider.crawl[:results]
+        put(
+          "follow::#{pattern}", spider.crawl
+        )
       }
     end
 

data/lib/spider_core/pagination_dsl.rb

@@ -3,10 +3,9 @@ module SpiderCore
 
     attr_accessor :next_page, :skip_pages
 
-    def keep_eyes_on_next_page(pattern, opts = {}, &block)
-      kind = opts[:kind] || :css
+    def keep_eyes_on_next_page(pattern, attr: :href, &block)
       actions << lambda {
-        element = first(kind, pattern)
+        element = first(pattern)
         path = block_given? ? yield(element) : element && element[:href]
         @paths.unshift(path) if path
       }

data/lib/spider_core/version.rb

@@ -1,3 +1,3 @@
 module SpiderCore
-  VERSION = "0.1.23"
+  VERSION = "0.2.0"
 end

data/test/micro_spider_test.rb

@@ -1,10 +1,10 @@
 require 'test_helper'
 
-class MicroSpiderTest < MiniTest::Unit::TestCase
+class MicroSpiderTest < Minitest::Unit::TestCase
 
   def setup
-   @spider = MicroSpider.new
-   @spider.logger.level = Logger::WARN
+    @spider = MicroSpider.new
+    @spider.logger.level = Logger::WARN
   end
 
   def test_spider_can_visit_path_with_some_delays
@@ -16,22 +16,36 @@ class MicroSpiderTest < MiniTest::Unit::TestCase
     assert (Time.now - now) > 5
   end
 
+  def test_spider_can_get_field
+    @spider.learn do
+      entrance '/'
+      entrance '/a'
+      field :name, '#name'
+    end
+    excretion = @spider.crawl
+    assert_equal 'Home', excretion['/']['name']
+    assert_equal 'This is a', excretion['/a']['name']
+    assert_includes @spider.get('name'), 'Home'
+    assert_includes @spider.get('name'), 'This is a'
+    assert_equal nil, @spider.get('name1')
+  end
+
   def test_spider_can_follow_lots_of_links
     @spider.entrance('/')
     @spider.follow('.links a') do
       field :name, '#name'
     end
     excretion = @spider.crawl
-    excretion[:results].first[:follow].first.each do |f|
-      case f[:entrance]
+    excretion['/']["follow::.links a"].each do |path, value|
+      case path
       when '/a'
-        assert_equal 'This is a', f[:field].first[:name]
+        assert_equal 'This is a', value.get('name')
       when '/b'
-        assert_equal 'This is b', f[:field].first[:name]
+        assert_equal 'This is b', value.get('name')
       when '/c'
-        assert_equal 'This is c', f[:field].first[:name]
+        assert_equal 'This is c', value.get('name')
       when '/d'
-        assert_equal 'This is d', f[:field].first[:name]
+        assert_equal 'This is d', value.get('name')
       end
     end
   end
@@ -44,18 +58,17 @@ class MicroSpiderTest < MiniTest::Unit::TestCase
       end
     end
     excretion = @spider.crawl
-    excretion[:results].first[:follow].first.each do |f|
-      refute_empty f[:follow].first
-      f[:follow].first.each do |ff|
-        case ff[:entrance]
+    excretion['/']["follow::.links a"].each do |key, value|
+      value["follow::.links a"].each do |k, v|
+        case k
         when '/a'
-          assert_equal 'This is a', ff[:field].first[:name]
+          assert_equal 'This is a', v.get('name')
         when '/b'
-          assert_equal 'This is b', ff[:field].first[:name]
+          assert_equal 'This is b', v.get('name')
         when '/c'
-          assert_equal 'This is c', ff[:field].first[:name]
+          assert_equal 'This is c', v.get('name')
         when '/d'
-          assert_equal 'This is d', ff[:field].first[:name]
+          assert_equal 'This is d', v.get('name')
         end
       end
     end
@@ -68,12 +81,12 @@ class MicroSpiderTest < MiniTest::Unit::TestCase
       field(:current_page, '#current_page')
     end
     excretion = @spider.crawl
-    excretion[:results].each do |f|
-      f[:entrance] =~ /\/page\/(\d)/
-      assert_equal "Current Page #{$1}", f[:field].first[:current_page]
+    excretion.each do |k,v|
+      k =~ /\/page\/(\d)/
+      assert_equal "Current Page #{$1}", v.get('current_page')
     end
   end
-  
+
   def test_spider_can_follow_and_keep_eyes_on_next_page
     @spider.entrance('/page/1')
     @spider.follow('a.next_page') do
@@ -81,9 +94,9 @@ class MicroSpiderTest < MiniTest::Unit::TestCase
       field :current_page, '#current_page'
     end
     excretion = @spider.crawl
-    excretion[:results].first[:follow].first.each do |f|
-      f[:entrance] =~ /\/page\/(\d)/
-      assert_equal "Current Page #{$1}", f[:field].first[:current_page]
+    excretion['/page/1']['follow::a.next_page'].each do |k, v|
+      k =~ /\/page\/(\d)/
+      assert_equal "Current Page #{$1}", v.get('current_page')
     end
   end
 
@@ -91,33 +104,37 @@ class MicroSpiderTest < MiniTest::Unit::TestCase
   end
 
   def test_spider_can_create_custom_action
+    @saved = false
     @spider.create_action(:save) do |result|
-      result[:save] = 'saved'
-    end
-    @spider.learn do
-      entrance '/'
-      field :name, '#name'
-      save
-    end
-    excretion = @spider.crawl
-    assert_equal 'saved', excretion[:results].first[:save]
-  end
-
-  def test_spider_can_create_custom_action_reached_by_spawn
-    @spider.create_action(:save) do |result|
-      result[:save] = 'saved'
+      @saved = true
     end
     @spider.learn do
       entrance '/'
       field :name, '#name'
       save
-      follow '.links a' do
-        field :name, '#name'
-        save
-      end
     end
     excretion = @spider.crawl
-    assert_equal 'saved', excretion[:results].first[:follow].first[0][:save]
+    assert_equal true, @saved
+    assert_equal 'Home', excretion['/']['name']
+    assert_equal 'Home', @spider.get('name')
   end
 
+  #def test_spider_can_create_custom_action_reached_by_spawn
+    #@saved = false
+    #@spider.create_action(:save) do |result|
+      #@saved = true
+    #end
+    #@spider.learn do
+      #entrance '/'
+      #field :name, '#name'
+      #save
+      #follow '.links a' do
+        #field :name, '#name'
+        #save
+      #end
+    #end
+    #excretion = @spider.crawl
+    #require 'pry'; binding.pry
+    #assert_equal true, @saved
+  #end
 end

data/test/test_helper.rb

@@ -7,7 +7,7 @@ rescue Bundler::BundlerError => e
 end
 
 require 'sinatra/base'
-require 'test/unit'
+require 'minitest/autorun'
 require 'pry'
 
 # Enable turn if it is available

metadata

@@ -1,144 +1,215 @@
 --- !ruby/object:Gem::Specification
 name: micro_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.23
-  prerelease: 
+  version: 0.2.0
 platform: ruby
 authors:
 - zires
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-10 00:00:00.000000000 Z
+date: 2016-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: capybara-mechanize
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
 - !ruby/object:Gem::Dependency
-  name: enumerable-lazy
+  name: hamster
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.4
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.4
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.5
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.7.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.7.5
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.7.5
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.3
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.3
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.3
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.7.6
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.7.6
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.7
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 11.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 11.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 11.2.2
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 11.2.0
 - !ruby/object:Gem::Dependency
   name: turn
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.7
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.7
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.7
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.7
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.7
 description: A DSL to write web spider. Depend on capybara and capybara-webkit.
 email:
 - zshuaibin@gmail.com
@@ -146,43 +217,43 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- MIT-LICENSE
+- README.md
+- Rakefile
 - lib/micro_spider.rb
+- lib/spider_core.rb
 - lib/spider_core/behavior.rb
 - lib/spider_core/exceptions.rb
+- lib/spider_core/excretion.rb
 - lib/spider_core/field_dsl.rb
 - lib/spider_core/follow_dsl.rb
 - lib/spider_core/pagination_dsl.rb
 - lib/spider_core/version.rb
-- lib/spider_core.rb
-- MIT-LICENSE
-- Rakefile
-- README.md
 - test/micro_spider_test.rb
 - test/test_helper.rb
 homepage: https://github.com/zires/micro-spider
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A DSL to write web spider.
 test_files:
 - test/micro_spider_test.rb