micro_spider 0.1.16

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f9363e70b57c95de9256ea2549cf2ee76c2669c0
+  data.tar.gz: 7f8b0bc18fde686058c2b426f84c05b26ba3a1b3
+SHA512:
+  metadata.gz: eb6a2ca107f788c95b4244b06de2ac8b7c94e983e0306dbcce71872cee37dd5946784cb900681fdd9d0ee35f6e82c2c6f7abbfed4916fabbdcc97a1ee27c849b
+  data.tar.gz: 06ea26fcfd3b53edbb461927772a50d4320525e89f9d7c19e250cc93ba33937362dd60d7d5467dd5d7a0c8950fa67e50c0a7d2df6f5fd5c81a51ca1a9e78c9cb

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 zires
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,4 @@
+tiny-spider
+===========
+
+web spider

data/Rakefile

@@ -0,0 +1,19 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'yard'
+YARD::Rake::YardocTask.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+task :default => :test

data/lib/micro_spider.rb

@@ -0,0 +1,155 @@
+require 'capybara'
+require 'capybara-webkit'
+require 'capybara/dsl'
+
+Capybara.run_server = false
+Capybara.current_driver = :webkit
+
+require 'logger'
+require 'spider_core'
+
+class MicroSpider
+
+  include Capybara::DSL
+  include SpiderCore::Behavior
+  include SpiderCore::FieldDSL
+  include SpiderCore::FollowDSL
+  include SpiderCore::PaginationDSL
+
+  attr_reader   :excretion, :paths, :delay, :current_location
+  attr_accessor :logger, :actions, :recipe, :skip_set_entrance
+
+  def initialize(excretion = nil)
+    @paths     = []
+    @actions   = []
+    @excretion = excretion || { status: 'inprogress', results: [] }
+    @logger    = Logger.new(STDOUT)
+  end
+
+  # The seconds between each two request.
+  #
+  # @param sec [Float]
+  def delay=(sec)
+    raise ArgumentError, 'Delay sec can not be a negative number.' if sec.to_f < 0
+    @delay = sec.to_f
+  end
+
+  # Visit the path.
+  #
+  # @param path [String] the path to visit, can be absolute path or relative path.
+  # @example Visit a path
+  #   spider = TinySpider.new
+  #   spider.visit('/example')
+  #   spider.visit('http://google.com')
+  #
+  def visit(path)
+    sleep_or_not
+    logger.info "Begin to visit #{path}."
+    super(path)
+    @current_location = {entrance: path}
+    logger.info "Current location is #{path}."
+  end
+
+  def click(locator, opts = {})
+    actions << lambda { 
+      path = find_link(locator, opts)[:href]
+      visit(path)
+    }
+  end
+
+  def learn(recipe = nil, &block)
+    if block_given?
+      instance_eval(&block)
+      @recipe = block
+    elsif recipe.is_a?(Proc)
+      instance_eval(&recipe)
+      @recipe = recipe
+    elsif recipe.is_a?(String)
+      instance_eval(recipe)
+      @recipe = recipe
+    else
+      self
+    end
+  end
+
+  def site(url)
+    return if @site
+    Capybara.app_host = @excretion[:site] = @site = url
+  end
+
+  def entrance(*path_or_paths)
+    return if @skip_set_entrance
+    @paths += path_or_paths
+  end
+
+  def entrance_on_path(path, pattern, kind: :css, **opts, &block)
+    return if @skip_set_entrance
+    visit(path)
+    entrances = scan_all(kind, pattern, opts).map do |element|
+      block_given? ? yield(element) : element[:href]
+    end
+    @paths += entrances.to_a
+  end
+
+  def crawl(&block)
+    return excretion if completed?
+
+    @paths.compact!
+    path = @paths.shift
+    if path.nil?
+      excretion[:status] = 'completed'
+      return excretion
+    end
+
+    visit(path)
+    execute_actions
+    yield(@current_location) if block_given?
+    excretion[:results] << @current_location
+
+    @skip_set_entrance = true
+    learn(@recipe)
+    crawl(&block)
+
+    excretion
+  end
+
+  def create_action(name, &block)
+    action = proc { actions << lambda { block.call(current_location) } }
+    metaclass.send :define_method, name, &action
+  end
+
+  def execute_actions
+    actions.delete_if { |action| action.call }
+  end
+
+  def spawn
+    spider = self.clone
+    spider.instance_variable_set(:@paths, [])
+    spider.instance_variable_set(:@actions, [])
+    spider.instance_variable_set(:@excretion, { status: 'inprogress', results: [] })
+    spider.skip_set_entrance = false
+    spider
+  end
+
+  def results
+    excretion[:results]
+  end
+
+  def completed?
+    excretion[:status] == 'completed'
+  end
+
+  def metaclass
+    class << self; self; end
+  end
+
+  protected
+  def sleep_or_not
+    if delay && delay > 0
+      logger.info "Nedd sleep #{delay} sec."
+      sleep(delay)
+      logger.info 'Wakeup'
+    end
+  end
+
+end

data/lib/spider_core/behavior.rb

@@ -0,0 +1,27 @@
+module SpiderCore
+  module Behavior
+
+    protected
+
+    def scan_all(kind, pattern, **opts)
+      if pattern.is_a?(String)
+        elements = all(kind, pattern).lazy
+        if opts[:limit] && opts[:limit].to_i > 0
+          elements = elements.take(opts[:limit].to_i)
+        end
+        return elements
+      elsif pattern.is_a?(Regexp)
+        html.scan(pattern).lazy
+      end
+    end
+
+    def scan_first(kind, pattern)
+      if pattern.is_a?(String)
+        first(kind, pattern)
+      elsif pattern.is_a?(Regexp)
+        html[pattern, 1]
+      end
+    end
+
+  end
+end

data/lib/spider_core/field_dsl.rb

@@ -0,0 +1,84 @@
+module SpiderCore
+  module FieldDSL
+
+    # Get a field on current page.
+    #
+    # @param display [String] display name
+    def field(display, pattern, opts = {}, &block)
+      kind = opts[:kind] || :css
+      actions << lambda {
+        action_for(:field, {display: display, pattern: pattern, kind: kind}, opts, &block)
+      }
+    end
+
+    def css_field(display, pattern, opts = {}, &block)
+      field(display, pattern, opts.merge(kind: :css), &block)
+    end
+
+    def xpath_field(display, pattern, opts = {}, &block)
+      field(display, pattern, opts.merge(kind: :xpath), &block)
+    end
+
+    def fields(display, pattern, opts = {}, &block)
+      kind = opts[:kind] || :css
+      actions << lambda {
+        action_for(:fields, {display: display, pattern: pattern, kind: kind}, opts, &block)
+      }
+    end
+
+    def css_fields(display, pattern, opts = {}, &block)
+      fields(display, pattern, opts.merge(kind: :css), &block)
+    end
+
+    def xpath_fields(display, pattern, opts = {}, &block)
+      fields(display, pattern, opts.merge(kind: :xpath), &block)
+    end
+
+    protected
+    def handle_element(element)
+      if element && element.respond_to?(:text)
+        element.text
+      else
+        element
+      end
+    end
+
+    def handle_elements(elements, &block)
+      if elements.respond_to?(:map) && block_given?
+        elements.map { |element| yield(element) }.force
+      elsif elements.respond_to?(:map)
+        elements.map { |element| handle_element(element) }.force
+      elsif block_given?
+        yield(elements)
+      else
+        handle_element(elements)
+      end
+    end
+
+    def action_for(action, action_opts = {}, opts = {}, &block)
+      begin
+        logger.info "Start to get `#{action_opts[:pattern]}` displayed `#{action_opts[:display]}`."
+
+        elements = case action
+        when :field
+          scan_first(action_opts[:kind], action_opts[:pattern])
+        when :fields
+          scan_all(action_opts[:kind], action_opts[:pattern], opts).lazy
+        else
+          raise 'Unknow action.'
+        end
+
+        make_field_result( action_opts[:display], handle_elements(elements, &block) )
+      rescue Exception => err
+        logger.fatal("Caught exception when get `#{action_opts[:pattern]}`.")
+        logger.fatal(err)
+      end
+    end
+
+    def make_field_result(display, field)
+      current_location[:field] ||= []
+      current_location[:field] << {display => field}
+    end
+
+  end
+end

data/lib/spider_core/follow_dsl.rb

@@ -0,0 +1,22 @@
+module SpiderCore
+  module FollowDSL
+
+    attr_accessor :skip_followers
+
+    def follow(pattern, kind: :css, **opts, &block)
+      return unless block_given?
+      actions << lambda {
+        spider = self.spawn
+        spider.learn(&block)
+        scan_all(kind, pattern, opts).each do |element|
+          next if skip_followers && skip_followers.include?(element[:href])
+          spider.skip_set_entrance = false
+          spider.entrance(element[:href])
+        end
+        current_location[:follow] ||= []
+        current_location[:follow] << spider.crawl[:results]
+      }
+    end
+
+  end
+end

data/lib/spider_core/pagination_dsl.rb

@@ -0,0 +1,14 @@
+module SpiderCore
+  module PaginationDSL
+
+    attr_accessor :next_page, :skip_pages
+
+    def keep_eyes_on_next_page(pattern, kind: :css)
+      actions << lambda {
+        @next_page = first(kind, pattern)[:href] rescue nil
+        @paths.unshift(@next_page) if @next_page
+      }
+    end
+
+  end
+end

data/lib/spider_core/version.rb

@@ -0,0 +1,3 @@
+module SpiderCore
+  VERSION = "0.1.16"
+end

data/lib/spider_core.rb

@@ -0,0 +1,4 @@
+require 'spider_core/behavior'
+require 'spider_core/follow_dsl'
+require 'spider_core/field_dsl'
+require 'spider_core/pagination_dsl'

data/test/micro_spider_test.rb

@@ -0,0 +1,106 @@
+require 'test_helper'
+
+class MicroSpiderTest < MiniTest::Unit::TestCase
+
+  def setup
+   @spider = MicroSpider.new
+  end
+
+  def test_spider_can_visit_path_with_some_delays
+    @spider.delay = 5
+    now = Time.now
+    @spider.visit('/')
+    @spider.visit('/')
+    assert_equal 5, @spider.instance_variable_get(:@delay)
+    assert (Time.now - now) > 5
+  end
+
+  def test_spider_can_follow_lots_of_links
+    @spider.entrance('/')
+    @spider.follow('.links a') do
+      field :name, '#name'
+    end
+    excretion = @spider.crawl
+    excretion[:results].first[:follow].first.each do |f|
+      case f[:entrance]
+      when '/a'
+        assert_equal 'This is a', f[:field].first[:name]
+      when '/b'
+        assert_equal 'This is b', f[:field].first[:name]
+      when '/c'
+        assert_equal 'This is c', f[:field].first[:name]
+      when '/d'
+        assert_equal 'This is d', f[:field].first[:name]
+      end
+    end
+  end
+
+  def test_spider_can_nest_follow_lots_of_links
+    @spider.entrance('/')
+    @spider.follow('.links a') do
+      follow('.links a') do
+        field :name, '#name'
+      end
+    end
+    excretion = @spider.crawl
+    excretion[:results].first[:follow].first.each do |f|
+      refute_empty f[:follow].first
+      f[:follow].first.each do |ff|
+        case ff[:entrance]
+        when '/a'
+          assert_equal 'This is a', ff[:field].first[:name]
+        when '/b'
+          assert_equal 'This is b', ff[:field].first[:name]
+        when '/c'
+          assert_equal 'This is c', ff[:field].first[:name]
+        when '/d'
+          assert_equal 'This is d', ff[:field].first[:name]
+        end
+      end
+    end
+  end
+
+  def test_spider_can_keep_eyes_on_next_page
+    @spider.entrance('/page/1')
+    @spider.learn do
+      keep_eyes_on_next_page('.pages a.next_page')
+      field(:current_page, '#current_page')
+    end
+    excretion = @spider.crawl
+    excretion[:results].each do |f|
+      f[:entrance] =~ /\/page\/(\d)/
+      assert_equal "Current Page #{$1}", f[:field].first[:current_page]
+    end
+  end
+
+  def test_spider_can_create_custom_action
+    @spider.create_action(:save) do |result|
+      result[:save] = 'saved'
+    end
+    @spider.learn do
+      entrance '/'
+      field :name, '#name'
+      save
+    end
+    excretion = @spider.crawl
+    assert_equal 'saved', excretion[:results].first[:save]
+  end
+
+  def test_spider_can_create_custom_action_reached_by_spawn
+    @spider.create_action(:save) do |result|
+      result[:save] = 'saved'
+    end
+    @spider.learn do
+      entrance '/'
+      field :name, '#name'
+      save
+      follow '.links a' do
+        field :name, '#name'
+        save
+      end
+    end
+    excretion = @spider.crawl
+    assert_equal 'saved', excretion[:results].first[:follow].first[0][:save]
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,103 @@
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'sinatra/base'
+require 'test/unit'
+require 'pry'
+
+# Enable turn if it is available
+begin
+  require 'turn'
+rescue LoadError
+end
+
+require 'micro_spider'
+
+class MyApp < Sinatra::Base
+
+  get '/' do
+    erb <<-ERB
+<div id="name">Home</div>
+<div class='links'>
+  <a href='/a'>A</a>
+  <a href='/b'>B</a>
+  <a href='/c'>C</a>
+  <a href='/d'>D</a>
+</div>
+    ERB
+  end
+
+  get '/a' do
+    erb <<-ERB
+<div id='name'>This is a</div>
+<div class='links'>
+  <a href='/a'>A</a>
+  <a href='/b'>B</a>
+  <a href='/c'>C</a>
+  <a href='/d'>D</a>
+</div>
+    ERB
+  end
+
+  get '/b' do
+    erb <<-ERB
+<div id='name'>This is b</div>
+<div class='links'>
+  <a href='/a'>A</a>
+  <a href='/b'>B</a>
+  <a href='/c'>C</a>
+  <a href='/d'>D</a>
+</div>
+    ERB
+  end
+
+
+  get '/c' do
+    erb <<-ERB
+<div id='name'>This is c</div>
+<div class='links'>
+  <a href='/a'>A</a>
+  <a href='/b'>B</a>
+  <a href='/c'>C</a>
+  <a href='/d'>D</a>
+</div>
+    ERB
+  end
+
+
+  get '/d' do
+    erb <<-ERB
+<div id='name'>This is d</div>
+<div class='links'>
+  <a href='/a'>A</a>
+  <a href='/b'>B</a>
+  <a href='/c'>C</a>
+  <a href='/d'>D</a>
+</div>
+    ERB
+  end
+
+  get '/page/:page' do
+    @current_page = params[:page]
+    erb <<-ERB
+<div id='current_page'>Current Page <%= @current_page %></div>
+<div class='pages'>
+  <% next_page = @current_page.to_i < 3 ? @current_page.to_i + 1 : nil %>
+  <% if next_page %>
+  <a href='/page/<%= next_page %>' class='next_page'>next page</a>
+  <% end %>
+</div>
+    ERB
+  end
+
+
+end
+
+Capybara.use_default_driver
+Capybara.app = MyApp
+

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: micro_spider
+version: !ruby/object:Gem::Version
+  version: 0.1.16
+platform: ruby
+authors:
+- zires
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara-webkit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A DSL to write web spider. Depend on capybara and capybara-webkit.
+email:
+- zshuaibin@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/micro_spider.rb
+- lib/spider_core/behavior.rb
+- lib/spider_core/field_dsl.rb
+- lib/spider_core/follow_dsl.rb
+- lib/spider_core/pagination_dsl.rb
+- lib/spider_core/version.rb
+- lib/spider_core.rb
+- MIT-LICENSE
+- Rakefile
+- README.md
+- test/micro_spider_test.rb
+- test/test_helper.rb
+homepage: https://github.com/zires/micro-spider
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0.rc.2
+signing_key: 
+specification_version: 4
+summary: A DSL to write web spider.
+test_files:
+- test/micro_spider_test.rb
+- test/test_helper.rb
+has_rdoc: