medusa-crawler 1.0.0.pre.1

data/lib/medusa/core.rb

@@ -0,0 +1,305 @@
+require 'thread'
+require 'robotex'
+require 'medusa/tentacle'
+require 'medusa/page'
+require 'medusa/exceptions'
+require 'medusa/page_store'
+require 'medusa/version'
+require 'medusa/storage'
+require 'medusa/storage/base'
+
+module Medusa
+  #
+  # Convenience method to start a crawl
+  #
+  def Medusa.crawl(urls, options = {}, &block)
+    Core.crawl(urls, options, &block)
+  end
+
+  class Core
+
+    # PageStore storing all Page objects encountered during the crawl
+    attr_reader :pages
+    # Hash of options for the crawl
+    attr_reader :opts
+
+    DEFAULT_OPTS = {
+      # run 4 Tentacle threads to fetch pages
+      :threads => 4,
+      # disable verbose output
+      :verbose => false,
+      # don't throw away the page response body after scanning it for links
+      :discard_page_bodies => false,
+      # identify self as Medusa/VERSION
+      :user_agent => "Medusa/#{Medusa::VERSION}",
+      # no delay between requests
+      :delay => 0,
+      # don't obey the robots exclusion protocol
+      :obey_robots_txt => false,
+      # by default, don't limit the depth of the crawl
+      :depth_limit => false,
+      # number of times HTTP redirects will be followed
+      :redirect_limit => 5,
+      # storage engine defaults to Hash in +process_options+ if none specified
+      :storage => nil,
+      # cleanups of the storage on every startup of the crawler
+      :clear_on_startup => true,
+      # Hash of cookie name => value to send with HTTP requests
+      :cookies => nil,
+      # accept cookies from the server and send them back?
+      :accept_cookies => false,
+      # skip any link with a query string? e.g. http://foo.com/?u=user
+      :skip_query_strings => false,
+      # proxy server hostname
+      :proxy_host => nil,
+      # proxy server port number
+      :proxy_port => false,
+      # HTTP read timeout in seconds
+      :read_timeout => nil
+    }.freeze
+
+    # Create setter methods for all options to be called from the crawl block
+    DEFAULT_OPTS.keys.each do |key|
+      define_method "#{key}=" do |value|
+        @opts[key.to_sym] = value
+      end
+    end
+
+    #
+    # Initialize the crawl with starting *urls* (single URL or Array of URLs)
+    # and optional *block*
+    #
+    def initialize(urls, opts = {})
+      @urls = [urls].flatten.map{ |url| url.is_a?(URI) ? url : URI(url) }
+      @urls.each{ |url| url.path = '/' if url.path.empty? }
+
+      @tentacles = []
+      @on_every_page_blocks = []
+      @on_pages_like_blocks = Hash.new { |hash,key| hash[key] = [] }
+      @skip_link_patterns = []
+      @after_crawl_blocks = []
+      @opts = opts
+      @focus_crawl_block = nil
+
+
+      yield self if block_given?
+    end
+
+    #
+    # Convenience method to start a new crawl
+    #
+    def self.crawl(urls, opts = {})
+      self.new(urls, opts) do |core|
+        yield core if block_given?
+        core.run
+      end
+    end
+
+    #
+    # Add a block to be executed on the PageStore after the crawl
+    # is finished
+    #
+    def after_crawl(&block)
+      @after_crawl_blocks << block
+      self
+    end
+
+    #
+    # Add one ore more Regex patterns for URLs which should not be
+    # followed
+    #
+    def skip_links_like(*patterns)
+      @skip_link_patterns.concat [patterns].flatten.compact
+      self
+    end
+
+    #
+    # Add a block to be executed on every Page as they are encountered
+    # during the crawl
+    #
+    def on_every_page(&block)
+      @on_every_page_blocks << block
+      self
+    end
+
+    #
+    # Add a block to be executed on Page objects with a URL matching
+    # one or more patterns
+    #
+    def on_pages_like(*patterns, &block)
+      if patterns
+        patterns.each do |pattern|
+          @on_pages_like_blocks[pattern] << block
+        end
+      end
+      self
+    end
+
+    #
+    # Specify a block which will select which links to follow on each page.
+    # The block should return an Array of URI objects.
+    #
+    def focus_crawl(&block)
+      @focus_crawl_block = block
+      self
+    end
+
+    #
+    # Perform the crawl
+    #
+    def run
+      process_options
+
+      @urls.delete_if { |url| !visit_link?(url) }
+      return if @urls.empty?
+
+      link_queue = Queue.new
+      page_queue = Queue.new
+
+      @opts[:threads].times do
+        @tentacles << Thread.new { Tentacle.new(link_queue, page_queue, @opts.dup).run }
+      end
+
+      @urls.each{ |url| link_queue.enq(url) }
+
+      loop do
+        page = page_queue.deq
+        @pages.touch_key page.url
+        puts "#{page.url} Queue: #{link_queue.size}" if @opts[:verbose]
+        do_page_blocks page
+        page.discard_doc! if @opts[:discard_page_bodies]
+
+        links = links_to_follow page
+        links.each do |link|
+          link_queue << [link, page.url.dup, page.depth + 1]
+        end
+        @pages.touch_keys links
+
+        @pages[page.url] = page
+
+        # if we are done with the crawl, tell the threads to end
+        if link_queue.empty? and page_queue.empty?
+          until link_queue.num_waiting == @tentacles.size
+            Thread.pass
+          end
+          if page_queue.empty?
+            @tentacles.size.times { link_queue << :END }
+            break
+          end
+        end
+      end
+
+      @tentacles.each { |thread| thread.join }
+      do_after_crawl_blocks
+      self
+    end
+
+    private
+
+    def process_options
+      @opts = DEFAULT_OPTS.merge @opts
+      @opts[:threads] = 1 if @opts[:delay] > 0
+      storage = Storage::Base.new(@opts[:storage] || Storage.Moneta(:Memory))
+      storage.clear if @opts[:clear_on_startup]
+      @pages = PageStore.new(storage)
+      @robots = Robotex.new(@opts[:user_agent]) if @opts[:obey_robots_txt]
+
+      freeze_options
+    end
+
+    #
+    # Freeze the opts Hash so that no options can be modified
+    # once the crawl begins
+    #
+    def freeze_options
+      @opts.freeze
+      @opts.each_key { |key| @opts[key].freeze }
+      @opts[:cookies].each_key { |key| @opts[:cookies][key].freeze } rescue nil
+    end
+
+    #
+    # Execute the after_crawl blocks
+    #
+    def do_after_crawl_blocks
+      @after_crawl_blocks.each { |block| block.call(@pages) }
+    end
+
+    #
+    # Execute the on_every_page blocks for *page*
+    #
+    def do_page_blocks(page)
+      @on_every_page_blocks.each do |block|
+        block.call(page)
+      end
+
+      @on_pages_like_blocks.each do |pattern, blocks|
+        blocks.each { |block| block.call(page) } if page.url.to_s =~ pattern
+      end
+    end
+
+    #
+    # Return an Array of links to follow from the given page.
+    # Based on whether or not the link has already been crawled,
+    # and the block given to focus_crawl()
+    #
+    def links_to_follow(page)
+      links = @focus_crawl_block ? @focus_crawl_block.call(page) : page.links
+      links.select { |link| visit_link?(link, page) }.map { |link| link.dup }
+    end
+
+    #
+    # Returns +true+ if *link* has not been visited already,
+    # and is not excluded by a skip_link pattern...
+    # and is not excluded by robots.txt...
+    # and is not deeper than the depth limit
+    # Returns +false+ otherwise.
+    #
+    def visit_link?(link, from_page = nil)
+      !@pages.has_page?(link) &&
+      !skip_link?(link) &&
+      !skip_query_string?(link) &&
+      allowed(link) &&
+      !too_deep?(from_page)
+    end
+
+    #
+    # Returns +true+ if we are obeying robots.txt and the link
+    # is granted access in it. Always returns +true+ when we are
+    # not obeying robots.txt.
+    #
+    def allowed(link)
+      @opts[:obey_robots_txt] ? @robots.allowed?(link) : true
+    rescue
+      false
+    end
+
+    #
+    # Returns +true+ if we are over the page depth limit.
+    # This only works when coming from a page and with the +depth_limit+ option set.
+    # When neither is the case, will always return +false+.
+    def too_deep?(from_page)
+      if from_page && @opts[:depth_limit]
+        from_page.depth >= @opts[:depth_limit]
+      else
+        false
+      end
+    end
+
+    #
+    # Returns +true+ if *link* should not be visited because
+    # it has a query string and +skip_query_strings+ is true.
+    #
+    def skip_query_string?(link)
+      @opts[:skip_query_strings] && link.query
+    end
+
+    #
+    # Returns +true+ if *link* should not be visited because
+    # its URL matches a skip_link pattern.
+    #
+    def skip_link?(link)
+      @skip_link_patterns.any? { |pattern| link.path =~ pattern }
+    end
+
+  end
+end

data/lib/medusa/exceptions.rb

@@ -0,0 +1,5 @@
+module Medusa
+  class Error < ::StandardError
+    attr_accessor :wrapped_exception
+  end
+end

data/lib/medusa/http.rb

@@ -0,0 +1,202 @@
+require 'rubygems'
+require 'medusa/page'
+require 'medusa/cookie_store'
+
+module Medusa
+  class HTTP
+    # Maximum number of redirects to follow on each get_response
+    REDIRECT_LIMIT = 5
+    RETRY_LIMIT = 6
+
+    # CookieStore for this HTTP client
+    attr_reader :cookie_store
+
+    def initialize(opts = {})
+      @opts = opts
+      @cookie_store = CookieStore.new(@opts[:cookies])
+    end
+
+    #
+    # Fetch a single Page from the response of an HTTP request to *url*.
+    # Just gets the final destination page.
+    #
+    def fetch_page(url, referer = nil, depth = nil)
+      fetch_pages(url, referer, depth).last
+    end
+
+    #
+    # Create new Pages from the response of an HTTP request to *url*,
+    # including redirects
+    #
+    def fetch_pages(url, referer = nil, depth = nil)
+      begin
+        url = URI(url) unless url.is_a?(URI)
+        pages = []
+        get(url, referer) do |response, headers, code, location, redirect_to, response_time|
+          pages << Page.new(location, :body => response,
+                                      :headers => headers,
+                                      :code => code,
+                                      :referer => referer,
+                                      :depth => depth,
+                                      :redirect_to => redirect_to,
+                                      :response_time => response_time)
+        end
+
+        return pages
+      rescue Exception => e
+        if verbose?
+          puts e.inspect
+          puts e.backtrace
+        end
+        pages ||= []
+        return pages << Page.new(url, :error => e)
+      end
+    end
+
+    #
+    # The maximum number of redirects to follow
+    #
+    def redirect_limit
+      @opts[:redirect_limit] || REDIRECT_LIMIT
+    end
+
+    #
+    # The user-agent string which will be sent with each request,
+    # or nil if no such option is set
+    #
+    def user_agent
+      @opts[:user_agent]
+    end
+
+    #
+    # Does this HTTP client accept cookies from the server?
+    #
+    def accept_cookies?
+      @opts[:accept_cookies]
+    end
+
+    #
+    # The http authentication options as in http://www.ruby-doc.org/stdlib/libdoc/open-uri/rdoc/OpenURI/OpenRead.html
+    # userinfo is deprecated [RFC3986]
+    #
+    def http_basic_authentication
+      @opts[:http_basic_authentication]
+    end
+
+    #
+    # The proxy authentication options as in http://www.ruby-doc.org/stdlib/libdoc/open-uri/rdoc/OpenURI/OpenRead.html
+    #
+    def proxy_http_basic_authentication
+      @opts[:proxy_http_basic_authentication]
+    end
+
+    #
+    # The proxy options as in http://www.ruby-doc.org/stdlib/libdoc/open-uri/rdoc/OpenURI/OpenRead.html
+    #
+    def proxy
+      @opts[:proxy]
+    end
+
+    #
+    # The proxy address string
+    #
+    def proxy_host
+      @opts[:proxy_host]
+    end
+
+    #
+    # The proxy port
+    #
+    def proxy_port
+      @opts[:proxy_port]
+    end
+
+    #
+    # HTTP read timeout in seconds
+    #
+    def read_timeout
+      @opts[:read_timeout]
+    end
+
+    private
+
+    #
+    # Retrieve HTTP responses for *url*, including redirects.
+    # Yields the response object, response code, and URI location
+    # for each response.
+    #
+    def get(url, referer = nil)
+      limit = redirect_limit
+      loc = url
+      begin
+          # if redirected to a relative url, merge it with the host of the original
+          # request url
+          loc = url.merge(loc) if loc.relative?
+
+          response, headers, response_time, response_code, redirect_to = get_response(loc, referer)
+
+          yield response, headers, Integer(response_code), loc, redirect_to, response_time
+          limit -= 1
+      end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
+    end
+
+    #
+    # Get an HTTPResponse for *url*, sending the appropriate User-Agent string
+    #
+    def get_response(url, referer = nil)
+      full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"
+
+      opts = {}
+      opts['User-Agent'] = user_agent if user_agent
+      opts['Referer'] = referer.to_s if referer
+      opts['Cookie'] = @cookie_store.to_s unless @cookie_store.empty? || (!accept_cookies? && @opts[:cookies].nil?)
+      opts[:http_basic_authentication] = http_basic_authentication if http_basic_authentication
+      opts[:proxy] = proxy if proxy
+      opts[:proxy_http_basic_authentication] = proxy_http_basic_authentication if proxy_http_basic_authentication
+      opts[:read_timeout] = read_timeout if !!read_timeout
+      opts[:redirect] = false
+      redirect_to = nil
+      retries = 0
+      begin
+        start = Time.now()
+
+        begin
+          if Gem::Requirement.new('< 2.5').satisfied_by?(Gem::Version.new(RUBY_VERSION))
+            resource = open(url, opts)
+          else
+            resource = URI.open(url, opts)
+          end
+        rescue OpenURI::HTTPRedirect => e_redirect
+          resource = e_redirect.io
+          redirect_to = e_redirect.uri
+        rescue OpenURI::HTTPError => e_http
+          resource = e_http.io
+        end
+
+        finish = Time.now()
+        response_time = ((finish - start) * 1000).round
+        @cookie_store.merge!(resource.meta['set-cookie']) if accept_cookies?
+        return resource.read, resource.meta, response_time, resource.status.shift, redirect_to
+
+      rescue Timeout::Error, EOFError, Errno::ECONNREFUSED, Errno::ETIMEDOUT, Errno::ECONNRESET => e
+        retries += 1
+        puts "[medusa] Retrying ##{retries} on url #{url} because of: #{e.inspect}" if verbose?
+        sleep(3 ^ retries)
+        retry unless retries > RETRY_LIMIT
+      ensure
+        resource.close if !resource.nil? && !resource.closed?
+      end
+    end
+
+    def verbose?
+      @opts[:verbose]
+    end
+
+    #
+    # Allowed to connect to the requested url?
+    #
+    def allowed?(to_url, from_url)
+      to_url.host.nil? || (to_url.host == from_url.host)
+    end
+  end
+end