micro_spider 0.1.16 → 0.1.17

data/lib/micro_spider.rb

@@ -2,10 +2,14 @@ require 'capybara'
 require 'capybara-webkit'
 require 'capybara/dsl'
 
-Capybara.run_server = false
 Capybara.current_driver = :webkit
+Capybara.configure do |config|
+  config.ignore_hidden_elements = false
+  config.run_server = false
+end
 
 require 'logger'
+require 'set'
 require 'spider_core'
 
 class MicroSpider
@@ -16,14 +20,16 @@ class MicroSpider
   include SpiderCore::FollowDSL
   include SpiderCore::PaginationDSL
 
-  attr_reader   :excretion, :paths, :delay, :current_location
+  attr_reader   :excretion, :paths, :delay, :current_location, :visited_paths, :broken_paths
   attr_accessor :logger, :actions, :recipe, :skip_set_entrance
 
   def initialize(excretion = nil)
-    @paths     = []
-    @actions   = []
+    @paths   = []
+    @actions = []
     @excretion = excretion || { status: 'inprogress', results: [] }
-    @logger    = Logger.new(STDOUT)
+    @logger        = Logger.new(STDOUT)
+    @visited_paths = Set.new
+    @broken_paths  = []
   end
 
   # The seconds between each two request.
@@ -37,8 +43,9 @@ class MicroSpider
   # Visit the path.
   #
   # @param path [String] the path to visit, can be absolute path or relative path.
+  #
   # @example Visit a path
-  #   spider = TinySpider.new
+  #   spider = MicroSpider.new
   #   spider.visit('/example')
   #   spider.visit('http://google.com')
   #
@@ -50,13 +57,37 @@ class MicroSpider
     logger.info "Current location is #{path}."
   end
 
+  # Click the locator. This will trigger visit action and change current location.
+  # @params locator [String] the text or id of the link.
+  #
   def click(locator, opts = {})
-    actions << lambda { 
+    actions << lambda {
       path = find_link(locator, opts)[:href]
       visit(path)
     }
   end
-
+  
+  # Teach the spider behaviors and it will repeat to the end.
+  # @param recipe [String, Proc] the recipe be learned.
+  #
+  # @example
+  #   spider = MicroSpider.new
+  #   spider.learn do
+  #     entrance 'http://google.com'
+  #   end
+  #   spider.crawl
+  #
+  # @example
+  #   spider.learn("entrance 'http://google.com'")
+  #   spider.crawl
+  #
+  # @example
+  #   recipe = lambda {
+  #     entrance 'http://google.com'
+  #   }
+  #   spider.learn(recipe)
+  #   spider.crawl
+  #
   def learn(recipe = nil, &block)
     if block_given?
       instance_eval(&block)
@@ -76,14 +107,33 @@ class MicroSpider
     return if @site
     Capybara.app_host = @excretion[:site] = @site = url
   end
-
+  
+  # This will be the first path for spider to visit.
+  # If more than one entrance, the spider will crawl theme one by one.
+  # @param path_or_paths [String] one or more entrances
+  #
+  # @example
+  #   spider = MicroSpider.new
+  #   spider.site('http://google.com')
+  #   spider.entrance('/a')
+  #   spider.entrance('/b')
+  #
   def entrance(*path_or_paths)
     return if @skip_set_entrance
     @paths += path_or_paths
   end
 
-  def entrance_on_path(path, pattern, kind: :css, **opts, &block)
+  # Sometimes the entrances are on the page.
+  # @param path [String] path to visit
+  # @param pattern [String, Regexp] links pattern
+  #
+  # @example
+  #   spider = MicroSpider.new
+  #   spider.entrance_on_path('http://google.com', '.links a')
+  #
+  def entrance_on_path(path, pattern, opts = {}, &block)
     return if @skip_set_entrance
+    kind = opts[:kind] || :css
     visit(path)
     entrances = scan_all(kind, pattern, opts).map do |element|
       block_given? ? yield(element) : element[:href]
@@ -95,24 +145,60 @@ class MicroSpider
     return excretion if completed?
 
     @paths.compact!
-    path = @paths.shift
+    path = nil
+    loop do
+      path = @paths.shift
+      break if path.nil?
+      break unless @visited_paths.include?(path)
+    end
+
     if path.nil?
       excretion[:status] = 'completed'
       return excretion
     end
 
-    visit(path)
-    execute_actions
-    yield(@current_location) if block_given?
-    excretion[:results] << @current_location
+    learn(@recipe) if @actions.empty?
 
-    @skip_set_entrance = true
-    learn(@recipe)
-    crawl(&block)
+    begin
+      visit(path)
+    rescue Timeout::Error => err
+      @broken_paths << path
+      logger.fatal("Timeout!!! execution expired when visit `#{path}`")
+      logger.fatal(err)
+    rescue SystemExit, Interrupt
+      logger.fatal("SystemExit && Interrupt")
+      exit!
+    rescue Exception => err
+      @broken_paths << path
+      logger.fatal("Caught exception when visit `#{path}`")
+      logger.fatal(err)
+    else
+      @visited_paths << path
+      execute_actions
+      yield(@current_location) if block_given?
+      excretion[:results] << @current_location
+    ensure
+      @actions = []
+      @skip_set_entrance = true
+      crawl(&block)
+    end
 
     excretion
   end
 
+  # Spider can create custom action when it is crawling.
+  # @param name [String] the name of action
+  # @param block [Proc] the actions
+  #
+  # @example
+  #   spider = MicroSpider.new
+  #
+  #   spider.create_action :save do |result|
+  #     SomeClass.save(result)
+  #   end
+  #
+  #   spider.save
+  #
   def create_action(name, &block)
     action = proc { actions << lambda { block.call(current_location) } }
     metaclass.send :define_method, name, &action
@@ -126,6 +212,8 @@ class MicroSpider
     spider = self.clone
     spider.instance_variable_set(:@paths, [])
     spider.instance_variable_set(:@actions, [])
+    spider.instance_variable_set(:@visited_paths, [])
+    spider.instance_variable_set(:@broken_paths, Set.new)
     spider.instance_variable_set(:@excretion, { status: 'inprogress', results: [] })
     spider.skip_set_entrance = false
     spider

data/lib/spider_core/behavior.rb

@@ -1,9 +1,11 @@
+require 'enumerable/lazy' if RUBY_VERSION < '2.0'
+
 module SpiderCore
   module Behavior
 
     protected
 
-    def scan_all(kind, pattern, **opts)
+    def scan_all(kind, pattern, opts = {})
       if pattern.is_a?(String)
         elements = all(kind, pattern).lazy
         if opts[:limit] && opts[:limit].to_i > 0

data/lib/spider_core/field_dsl.rb

@@ -36,10 +36,12 @@ module SpiderCore
 
     protected
     def handle_element(element)
-      if element && element.respond_to?(:text)
-        element.text
-      else
+      if element.is_a?(String)
         element
+      elsif element.tag_name == 'input'
+        element.value
+      else
+        element.text
       end
     end
 
@@ -63,7 +65,7 @@ module SpiderCore
         when :field
           scan_first(action_opts[:kind], action_opts[:pattern])
         when :fields
-          scan_all(action_opts[:kind], action_opts[:pattern], opts).lazy
+          scan_all(action_opts[:kind], action_opts[:pattern], opts)
         else
           raise 'Unknow action.'
         end

data/lib/spider_core/follow_dsl.rb

@@ -3,8 +3,9 @@ module SpiderCore
 
     attr_accessor :skip_followers
 
-    def follow(pattern, kind: :css, **opts, &block)
+    def follow(pattern, opts = {}, &block)
       return unless block_given?
+      kind = opts[:kind] || :css
       actions << lambda {
         spider = self.spawn
         spider.learn(&block)

data/lib/spider_core/pagination_dsl.rb

@@ -3,7 +3,8 @@ module SpiderCore
 
     attr_accessor :next_page, :skip_pages
 
-    def keep_eyes_on_next_page(pattern, kind: :css)
+    def keep_eyes_on_next_page(pattern, opts = {})
+      kind = opts[:kind] || :css
       actions << lambda {
         @next_page = first(kind, pattern)[:href] rescue nil
         @paths.unshift(@next_page) if @next_page

data/lib/spider_core/version.rb

@@ -1,3 +1,3 @@
 module SpiderCore
-  VERSION = "0.1.16"
+  VERSION = "0.1.17"
 end

data/test/micro_spider_test.rb

@@ -4,6 +4,7 @@ class MicroSpiderTest < MiniTest::Unit::TestCase
 
   def setup
    @spider = MicroSpider.new
+   @spider.logger.level = Logger::WARN
   end
 
   def test_spider_can_visit_path_with_some_delays

metadata

@@ -1,111 +1,142 @@
 --- !ruby/object:Gem::Specification
 name: micro_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.16
+  version: 0.1.17
+  prerelease: 
 platform: ruby
 authors:
 - zires
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-23 00:00:00.000000000 Z
+date: 2013-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: capybara-webkit
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: enumerable-lazy
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: turn
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: A DSL to write web spider. Depend on capybara and capybara-webkit.
@@ -128,27 +159,29 @@ files:
 - test/micro_spider_test.rb
 - test/test_helper.rb
 homepage: https://github.com/zires/micro-spider
-licenses: []
-metadata: {}
+licenses:
+- MIT
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0.rc.2
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: A DSL to write web spider.
 test_files:
 - test/micro_spider_test.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: f9363e70b57c95de9256ea2549cf2ee76c2669c0
-  data.tar.gz: 7f8b0bc18fde686058c2b426f84c05b26ba3a1b3
-SHA512:
-  metadata.gz: eb6a2ca107f788c95b4244b06de2ac8b7c94e983e0306dbcce71872cee37dd5946784cb900681fdd9d0ee35f6e82c2c6f7abbfed4916fabbdcc97a1ee27c849b
-  data.tar.gz: 06ea26fcfd3b53edbb461927772a50d4320525e89f9d7c19e250cc93ba33937362dd60d7d5467dd5d7a0c8950fa67e50c0a7d2df6f5fd5c81a51ca1a9e78c9cb