medusa-crawler 1.0.0.pre.1

data/lib/medusa/page.rb

@@ -0,0 +1,229 @@
+require 'rubygems'
+require 'cgi'
+require 'nokogiri'
+require 'ostruct'
+require 'webrick/cookie'
+
+module Medusa
+  class Page
+
+    # The URL of the page
+    attr_reader :url
+    # The raw HTTP response body of the page
+    attr_reader :body
+    # Headers of the HTTP response
+    attr_reader :headers
+    # URL of the page this one redirected to, if any
+    attr_reader :redirect_to
+    # Exception object, if one was raised during HTTP#fetch_page
+    attr_reader :error
+
+    # OpenStruct for user-stored data
+    attr_accessor :data
+    # Integer response code of the page
+    attr_accessor :code
+    # Boolean indicating whether or not this page has been visited in PageStore#shortest_paths!
+    attr_accessor :visited
+    # Depth of this page from the root of the crawl. This is not necessarily the
+    # shortest path; use PageStore#shortest_paths! to find that value.
+    attr_accessor :depth
+    # URL of the page that brought us to this page
+    attr_accessor :referer
+    # Response time of the request for this page in milliseconds
+    attr_accessor :response_time
+
+    #
+    # Create a new page
+    #
+    def initialize(url, params = {})
+      @url = url
+      @data = OpenStruct.new
+
+      @links = nil
+      @visited = false
+      @body = nil
+      @doc = nil
+      @base = nil
+
+      @code = params[:code]
+      @headers = params[:headers] || {}
+      @headers['content-type'] ||= ''
+      @aliases = Array(params[:aka]).compact
+      @referer = params[:referer]
+      @depth = params[:depth] || 0
+      @redirect_to = to_absolute(params[:redirect_to])
+      @response_time = params[:response_time]
+      @body = params[:body]
+      @error = params[:error]
+
+      @fetched = !params[:code].nil?
+    end
+
+    #
+    # Array of distinct A tag HREFs from the page
+    #
+    def links
+      return @links unless @links.nil?
+      @links = []
+      return @links if !doc
+
+      doc.search("//a[@href]").each do |a|
+        next if a['data-method'] && a['data-method'] != 'get'
+        u = a['href']
+        next if u.nil? or u.empty?
+        abs = to_absolute(u) rescue next
+        @links << abs if in_domain?(abs)
+      end
+      @links.uniq!
+      @links
+    end
+
+    #
+    # Nokogiri document for the HTML body
+    #
+    def doc
+      return @doc if @doc
+      @doc = Nokogiri::HTML(@body) if @body && html? rescue nil
+    end
+
+    #
+    # Delete the Nokogiri document and response body to conserve memory
+    #
+    def discard_doc!
+      links # force parsing of page links before we trash the document
+      @doc = @body = nil
+    end
+
+    #
+    # Was the page successfully fetched?
+    # +true+ if the page was fetched with no error, +false+ otherwise.
+    #
+    def fetched?
+      @fetched
+    end
+
+    #
+    # Array of cookies received with this page as WEBrick::Cookie objects.
+    #
+    def cookies
+      WEBrick::Cookie.parse_set_cookies(@headers['set-cookie']) rescue []
+    end
+
+    #
+    # The content-type returned by the HTTP request for this page
+    #
+    def content_type
+      headers['content-type']
+    end
+
+    #
+    # Returns +true+ if the page is a HTML document, returns +false+
+    # otherwise.
+    #
+    def html?
+      !!(content_type =~ %r{^(text/html|application/xhtml+xml)\b})
+    end
+
+    #
+    # Returns +true+ if the page is a HTTP redirect, returns +false+
+    # otherwise.
+    #
+    def redirect?
+      (300..307).include?(@code)
+    end
+
+    #
+    # Returns +true+ if the page was not found (returned 404 code),
+    # returns +false+ otherwise.
+    #
+    def not_found?
+      404 == @code
+    end
+
+    #
+    # Base URI from the HTML doc head element
+    # http://www.w3.org/TR/html4/struct/links.html#edef-BASE
+    #
+    def base
+      @base = if doc
+        href = doc.search('//head/base/@href')
+        URI(href.to_s) unless href.nil? rescue nil
+      end unless @base
+
+      return nil if @base && @base.to_s().empty?
+      @base
+    end
+
+
+    #
+    # Converts relative URL *link* into an absolute URL based on the
+    # location of the page
+    #
+    def to_absolute(link)
+      return nil if link.nil?
+
+      # remove anchor
+      link = link.to_s.gsub(/#.*$/,'')
+      if Gem::Requirement.new('< 2.5').satisfied_by?(Gem::Version.new(RUBY_VERSION))
+        link = URI.encode(URI.decode(link))
+      end
+
+      relative = URI(link)
+      absolute = base ? base.merge(relative) : @url.merge(relative)
+
+      absolute.path = '/' if absolute.path.empty?
+
+      return absolute
+    end
+
+    #
+    # Returns +true+ if *uri* is in the same domain as the page, returns
+    # +false+ otherwise
+    #
+    def in_domain?(uri)
+      uri.host == @url.host
+    end
+
+    def marshal_dump
+      [@url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched]
+    end
+
+    def marshal_load(ary)
+      @url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched = ary
+    end
+
+    def to_hash
+      {'url' => @url.to_s,
+       'headers' => Marshal.dump(@headers),
+       'data' => Marshal.dump(@data),
+       'body' => @body,
+       'links' => links.map(&:to_s),
+       'code' => @code,
+       'visited' => @visited,
+       'depth' => @depth,
+       'referer' => @referer.to_s,
+       'redirect_to' => @redirect_to.to_s,
+       'response_time' => @response_time,
+       'fetched' => @fetched}
+    end
+
+    def self.from_hash(hash)
+      page = self.new(URI(hash['url']))
+      {'@headers' => Marshal.load(hash['headers']),
+       '@data' => Marshal.load(hash['data']),
+       '@body' => hash['body'],
+       '@links' => hash['links'].map { |link| URI(link) },
+       '@code' => hash['code'].to_i,
+       '@visited' => hash['visited'],
+       '@depth' => hash['depth'].to_i,
+       '@referer' => hash['referer'],
+       '@redirect_to' => (!!hash['redirect_to'] && !hash['redirect_to'].empty?) ? URI(hash['redirect_to']) : nil,
+       '@response_time' => hash['response_time'].to_i,
+       '@fetched' => hash['fetched']
+      }.each do |var, value|
+        page.instance_variable_set(var, value)
+      end
+      page
+    end
+  end
+end

data/lib/medusa/page_store.rb

@@ -0,0 +1,160 @@
+require 'forwardable'
+
+module Medusa
+  class PageStore
+    extend Forwardable
+
+    def_delegators :@storage, :keys, :values, :size, :each
+
+    def initialize(storage = {})
+      @storage = storage
+    end
+
+    # We typically index the hash with a URI,
+    # but convert it to a String for easier retrieval
+    def [](index)
+      @storage[index.to_s]
+    end
+
+    def []=(index, other)
+      @storage[index.to_s] = other
+    end
+
+    def delete(key)
+      @storage.delete key.to_s
+    end
+
+    def has_key?(key)
+      @storage.has_key? key.to_s
+    end
+
+    def each_value
+      each { |key, value| yield value }
+    end
+
+    def values
+      result = []
+      each { |key, value| result << value }
+      result
+    end
+
+    def touch_key(key)
+      self[key] = Page.new(key)
+    end
+
+    def touch_keys(keys)
+      @storage.merge! keys.inject({}) { |h, k| h[k.to_s] = Page.new(k); h }
+    end
+
+    # Does this PageStore contain the specified URL?
+    # HTTP and HTTPS versions of a URL are considered to be the same page.
+    def has_page?(url)
+      schemes = %w(http https)
+      if schemes.include? url.scheme
+        u = url.dup
+        return schemes.any? { |s| u.scheme = s; has_key?(u) }
+      end
+
+      has_key? url
+    end
+
+    #
+    # Use a breadth-first search to calculate the single-source
+    # shortest paths from *root* to all pages in the PageStore
+    #
+    def shortest_paths!(root)
+      root = URI(root) if root.is_a?(String)
+      raise "Root node not found" if !has_key?(root)
+
+      q = Queue.new
+
+      q.enq root
+      root_page = self[root]
+      root_page.depth = 0
+      root_page.visited = true
+      self[root] = root_page
+      while !q.empty?
+        page = self[q.deq]
+        page.links.each do |u|
+          begin
+            link = self[u]
+            next if link.nil? || !link.fetched? || link.visited
+
+            q << u unless link.redirect?
+            link.visited = true
+            link.depth = page.depth + 1
+            self[u] = link
+
+            if link.redirect?
+              u = link.redirect_to
+              redo
+            end
+          end
+        end
+      end
+
+      self
+    end
+
+    #
+    # Removes all Pages from storage where redirect? is true
+    #
+    def uniq!
+      each_value { |page| delete page.url if page.redirect? }
+      self
+    end
+
+    #
+    # If given a single URL (as a String or URI), returns an Array of Pages which link to that URL
+    # If given an Array of URLs, returns a Hash (URI => [Page, Page...]) of Pages linking to those URLs
+    #
+    def pages_linking_to(urls)
+      unless urls.is_a?(Array)
+        urls = [urls]
+        single = true
+      end
+
+      urls.map! do |url|
+        unless url.is_a?(URI)
+          URI(url) rescue nil
+        else
+          url
+        end
+      end
+      urls.compact
+
+      links = {}
+      urls.each { |url| links[url] = [] }
+      values.each do |page|
+        urls.each { |url| links[url] << page if page.links.include?(url) }
+      end
+
+      if single and !links.empty?
+        return links[urls.first]
+      else
+        return links
+      end
+    end
+
+    #
+    # If given a single URL (as a String or URI), returns an Array of URLs which link to that URL
+    # If given an Array of URLs, returns a Hash (URI => [URI, URI...]) of URLs linking to those URLs
+    #
+    def urls_linking_to(urls)
+      unless urls.is_a?(Array)
+        urls = [urls] unless urls.is_a?(Array)
+        single = true
+      end
+
+      links = pages_linking_to(urls)
+      links.each { |url, pages| links[url] = pages.map{|p| p.url} }
+
+      if single and !links.empty?
+        return links[urls.first]
+      else
+        return links
+      end
+    end
+
+  end
+end

data/lib/medusa/storage.rb

@@ -0,0 +1,8 @@
+module Medusa
+  module Storage
+    def self.Moneta(*args)
+      require 'medusa/storage/moneta'
+      self::Moneta.new(*args)
+    end
+  end
+end

data/lib/medusa/storage/base.rb

@@ -0,0 +1,81 @@
+require 'medusa/storage/exceptions'
+
+module Medusa
+  module Storage
+    class Base
+
+      def initialize(adapter)
+        @adap = adapter
+
+        # verify adapter conforms to this class's methods
+        public_methods(false).each do |method|
+          if !@adap.respond_to?(method.to_sym)
+            raise "Storage adapter must support method #{method}"
+          end
+        end
+      end
+
+      def [](key)
+        @adap[key]
+        rescue
+          puts key
+          raise RetrievalError
+      end
+
+      def []=(key, value)
+        @adap[key] = value
+        rescue
+          raise InsertionError
+      end
+
+      def delete(key)
+        @adap.delete(key)
+        rescue
+          raise DeletionError
+      end
+
+      def clear
+        @adap.clear
+        rescue
+          raise GenericError
+      end
+
+      def each
+        @adap.each { |k, v| yield k, v }
+        rescue
+          raise GenericError
+      end
+
+      def merge!(hash)
+        @adap.merge!(hash)
+        rescue
+          raise GenericError
+      end
+
+      def close
+        @adap.close
+        rescue
+          raise CloseError
+      end
+
+      def size
+        @adap.size
+        rescue
+          raise GenericError
+      end
+
+      def keys
+        @adap.keys
+        rescue
+          raise GenericError
+      end
+
+      def has_key?(key)
+        @adap.has_key?(key)
+        rescue
+          raise GenericError
+      end
+
+    end
+  end
+end