micro_spider 0.1.17 → 0.1.18

data/README.md

@@ -1,4 +1,33 @@
-tiny-spider
-===========
+# micro-spider
+
+A DSL to write web spider. Depend on capybara and capybara-webkit.
+
+# Example
+
+```ruby
+require 'micro_spider'
+spider = MicroSpider.new
+
+spider.learn do
+  site 'http://www.bbc.com'
+  entrance '/news'
+  
+  field :top_story, '#top-story h2 a'
+  
+  follow '.story' do
+    
+    field :title, 'h1.story-header'
+    field :body,  '.story-body'
+
+    fields :related_stories, '.related-links-list a'
+    
+  end
+  
+end
+
+spider.crawl
+
+```
+
+
 
-web spider

data/lib/micro_spider.rb

@@ -10,6 +10,7 @@ end
 
 require 'logger'
 require 'set'
+require 'timeout'
 require 'spider_core'
 
 class MicroSpider
@@ -21,11 +22,12 @@ class MicroSpider
   include SpiderCore::PaginationDSL
 
   attr_reader   :excretion, :paths, :delay, :current_location, :visited_paths, :broken_paths
-  attr_accessor :logger, :actions, :recipe, :skip_set_entrance
+  attr_accessor :logger, :actions, :recipe, :skip_set_entrance, :timeout
 
   def initialize(excretion = nil)
     @paths   = []
     @actions = []
+    @timeout = 120
     @excretion = excretion || { status: 'inprogress', results: [] }
     @logger        = Logger.new(STDOUT)
     @visited_paths = Set.new
@@ -60,13 +62,21 @@ class MicroSpider
   # Click the locator. This will trigger visit action and change current location.
   # @params locator [String] the text or id of the link.
   #
-  def click(locator, opts = {})
+  def click(locator, opts = {}, &block)
     actions << lambda {
       path = find_link(locator, opts)[:href]
-      visit(path)
+      if block_given?
+        spider = self.spawn
+        spider.entrance(path)
+        spider.learn(&block)
+        current_location[:click] ||= []
+        current_location[:click] << spider.crawl[:results]
+      else
+        visit(path)
+      end
     }
   end
-  
+
   # Teach the spider behaviors and it will repeat to the end.
   # @param recipe [String, Proc] the recipe be learned.
   #
@@ -205,14 +215,23 @@ class MicroSpider
   end
 
   def execute_actions
-    actions.delete_if { |action| action.call }
+    actions.delete_if { |action|
+      begin
+        Timeout::timeout(@timeout) { action.call }
+      rescue Timeout::Error => err
+        logger.fatal('Timeout!!! execution expired when execute action')
+        logger.fatal(err.message)
+        logger.fatal(err.backtrace.inspect)
+        break
+      end
+    }
   end
 
   def spawn
     spider = self.clone
     spider.instance_variable_set(:@paths, [])
     spider.instance_variable_set(:@actions, [])
-    spider.instance_variable_set(:@visited_paths, [])
+    spider.instance_variable_set(:@visited_paths, Set.new)
     spider.instance_variable_set(:@broken_paths, Set.new)
     spider.instance_variable_set(:@excretion, { status: 'inprogress', results: [] })
     spider.skip_set_entrance = false

data/lib/spider_core/pagination_dsl.rb

@@ -3,11 +3,12 @@ module SpiderCore
 
     attr_accessor :next_page, :skip_pages
 
-    def keep_eyes_on_next_page(pattern, opts = {})
+    def keep_eyes_on_next_page(pattern, opts = {}, &block)
       kind = opts[:kind] || :css
       actions << lambda {
-        @next_page = first(kind, pattern)[:href] rescue nil
-        @paths.unshift(@next_page) if @next_page
+        element = first(kind, pattern)
+        path = block_given? ? yield(element) : element && element[:href]
+        @paths.unshift(path) if path
       }
     end
 

data/lib/spider_core/version.rb

@@ -1,3 +1,3 @@
 module SpiderCore
-  VERSION = "0.1.17"
+  VERSION = "0.1.18"
 end

data/test/micro_spider_test.rb

@@ -73,6 +73,22 @@ class MicroSpiderTest < MiniTest::Unit::TestCase
       assert_equal "Current Page #{$1}", f[:field].first[:current_page]
     end
   end
+  
+  def test_spider_can_follow_and_keep_eyes_on_next_page
+    @spider.entrance('/page/1')
+    @spider.follow('a.next_page') do
+      keep_eyes_on_next_page('.pages a.next_page')
+      field :current_page, '#current_page'
+    end
+    excretion = @spider.crawl
+    excretion[:results].first[:follow].first.each do |f|
+      f[:entrance] =~ /\/page\/(\d)/
+      assert_equal "Current Page #{$1}", f[:field].first[:current_page]
+    end
+  end
+
+  def test_spider_can_nest_follow_lots_of_links_and_keep_eyes_on_next_page
+  end
 
   def test_spider_can_create_custom_action
     @spider.create_action(:save) do |result|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: micro_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.17
+  version: 0.1.18
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-25 00:00:00.000000000 Z
+date: 2013-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara