polipus 0.0.1

data/lib/polipus/http.rb

@@ -0,0 +1,195 @@
+require "net/https"
+require "polipus/page"
+require "zlib"
+require 'http/cookie'
+
+module Polipus
+  class HTTP
+    # Maximum number of redirects to follow on each get_response
+    REDIRECT_LIMIT = 5
+
+    def initialize(opts = {})
+      @connections = {}
+      @opts = opts
+    end
+
+    #
+    # Fetch a single Page from the response of an HTTP request to *url*.
+    # Just gets the final destination page.
+    #
+    def fetch_page(url, referer = nil, depth = nil)
+      fetch_pages(url, referer, depth).last
+    end
+
+    #
+    # Create new Pages from the response of an HTTP request to *url*,
+    # including redirects
+    #
+    def fetch_pages(url, referer = nil, depth = nil)
+      begin
+        url = URI(url) unless url.is_a?(URI)
+        pages = []
+        get(url, referer) do |response, code, location, redirect_to, response_time|
+          body = response.body.dup
+          if response.to_hash.fetch('content-encoding', [])[0] == 'gzip'
+            gzip = Zlib::GzipReader.new(StringIO.new(body))    
+            body = gzip.read
+          end
+          pages << Page.new(location, :body => response.body.dup,
+                                      :code => code,
+                                      :headers => response.to_hash,
+                                      :referer => referer,
+                                      :depth => depth,
+                                      :redirect_to => redirect_to,
+                                      :response_time => response_time)
+        end
+
+        return pages
+      rescue StandardError => e
+        if verbose?
+          puts e.inspect
+          puts e.backtrace
+        end
+        return [Page.new(url, :error => e)]
+      end
+    end
+
+    #
+    # The maximum number of redirects to follow
+    #
+    def redirect_limit
+      @opts[:redirect_limit] || REDIRECT_LIMIT
+    end
+
+    #
+    # The user-agent string which will be sent with each request,
+    # or nil if no such option is set
+    #
+    def user_agent
+      @opts[:user_agent]
+    end
+
+   
+    #
+    # The proxy address string
+    #
+    def proxy_host
+      @opts[:proxy_host]
+    end
+
+    #
+    # The proxy port
+    #
+    def proxy_port
+      @opts[:proxy_port]
+    end
+
+    #
+    # HTTP read timeout in seconds
+    #
+    def read_timeout
+      @opts[:read_timeout]
+    end
+
+    # Does this HTTP client accept cookies from the server?
+    #
+    def accept_cookies?
+      @opts[:accept_cookies]
+    end
+
+    def cookie_jar
+      @opts[:cookie_jar] ||= ::HTTP::CookieJar.new
+      @opts[:cookie_jar]
+    end
+
+    private
+
+    #
+    # Retrieve HTTP responses for *url*, including redirects.
+    # Yields the response object, response code, and URI location
+    # for each response.
+    #
+    def get(url, referer = nil)
+      limit = redirect_limit
+      loc = url
+      begin
+          # if redirected to a relative url, merge it with the host of the original
+          # request url
+          loc = url.merge(loc) if loc.relative?
+
+          response, response_time = get_response(loc, referer)
+          code = Integer(response.code)
+          redirect_to = response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
+          yield response, code, loc, redirect_to, response_time
+          limit -= 1
+      end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
+    end
+
+    #
+    # Get an HTTPResponse for *url*, sending the appropriate User-Agent string
+    #
+    def get_response(url, referer = nil)
+      full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"
+
+      opts = {}
+      opts['User-Agent'] = user_agent if user_agent
+      opts['Referer'] = referer.to_s if referer
+      opts['Cookie']  = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
+
+      retries = 0
+      begin
+        start = Time.now()
+        # format request
+        req = Net::HTTP::Get.new(full_path, opts)
+        # HTTP Basic authentication
+        req.basic_auth url.user, url.password if url.user
+        response = connection(url).request(req)
+        finish = Time.now()
+        response_time = ((finish - start) * 1000).round
+        cookie_jar.parse(response["Set-Cookie"], url) if accept_cookies?
+        return response, response_time
+      rescue Timeout::Error, Net::HTTPBadResponse, EOFError => e
+        
+        puts e.inspect if verbose?
+        refresh_connection(url)
+        retries += 1
+        retry unless retries > 3
+      end
+    end
+
+    def connection(url)
+      @connections[url.host] ||= {}
+
+      if conn = @connections[url.host][url.port]
+        return conn
+      end
+
+      refresh_connection url
+    end
+
+    def refresh_connection(url)
+      http = Net::HTTP.new(url.host, url.port, proxy_host, proxy_port)
+
+      http.read_timeout = read_timeout if !!read_timeout
+
+      if url.scheme == 'https'
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      @connections[url.host][url.port] = http.start 
+    end
+
+    def verbose?
+      @opts[:verbose]
+    end
+
+    #
+    # Allowed to connect to the requested url?
+    #
+    def allowed?(to_url, from_url)
+      to_url.host.nil? || (to_url.host == from_url.host)
+    end
+
+  end
+end

data/lib/polipus/page.rb

@@ -0,0 +1,219 @@
+require 'nokogiri'
+require 'json'
+require 'ostruct'
+require 'set'
+module Polipus
+  class Page
+
+    # The URL of the page
+    attr_reader :url
+    # The raw HTTP response body of the page
+    attr_reader :body
+    # Headers of the HTTP response
+    attr_reader :headers
+    # URL of the page this one redirected to, if any
+    attr_reader :redirect_to
+    # Exception object, if one was raised during HTTP#fetch_page
+    attr_reader :error
+    # Integer response code of the page
+    attr_accessor :code
+    # Depth of this page from the root of the crawl. This is not necessarily the
+    # shortest path; use PageStore#shortest_paths! to find that value.
+    attr_accessor :depth
+    # URL of the page that brought us to this page
+    attr_accessor :referer
+    # Response time of the request for this page in milliseconds
+    attr_accessor :response_time
+    # OpenStruct it holds users defined data
+    attr_accessor :user_data
+
+    attr_accessor :aliases
+
+    attr_accessor :domain_aliases
+
+    #
+    # Create a new page
+    #
+    def initialize(url, params = {})
+      @url = url.kind_of?(URI) ? url : URI(url)
+      @code = params[:code]
+      @headers = params[:headers] || {}
+      @headers['content-type'] ||= ['']
+      @aliases = Array(params[:aka]).compact
+      @referer = params[:referer]
+      @depth = params[:depth] || 0
+      @redirect_to = to_absolute(params[:redirect_to])
+      @response_time = params[:response_time]
+      @body = params[:body]
+      @error = params[:error]
+      @fetched = !params[:code].nil?
+      @user_data = OpenStruct.new
+      @domain_aliases = params[:domain_aliases] ||= []
+    end
+
+    #
+    # Array of distinct A tag HREFs from the page
+    #
+    def links
+      return @links.to_a unless @links.nil?
+      @links = Set.new
+      return [] if !doc
+      
+      doc.search("//a[@href]").each do |a|
+        u = a['href']
+        next if u.nil? or u.empty?
+        abs = to_absolute(u) rescue next
+        @links << abs if in_domain?(abs)
+      end
+      @links.to_a
+    end
+
+    #
+    # Nokogiri document for the HTML body
+    #
+    def doc
+      return @doc if @doc
+      @doc = Nokogiri::HTML(@body) if @body && html? rescue nil
+    end
+
+    #
+    # Discard links, a next call of page.links will return an empty array
+    #
+    def discard_links!
+      @links = []
+    end
+
+    #
+    # Delete the Nokogiri document and response body to conserve memory
+    #
+    def discard_doc!
+      links # force parsing of page links before we trash the document
+      @doc = @body = nil
+    end
+
+    #
+    # Was the page successfully fetched?
+    # +true+ if the page was fetched with no error, +false+ otherwise.
+    #
+    def fetched?
+      @fetched
+    end
+
+    #
+    # The content-type returned by the HTTP request for this page
+    #
+    def content_type
+      headers['content-type'].first
+    end
+
+    #
+    # Returns +true+ if the page is a HTML document, returns +false+
+    # otherwise.
+    #
+    def html?
+      !!(content_type =~ %r{^(text/html|application/xhtml+xml)\b})
+    end
+
+    #
+    # Returns +true+ if the page is a HTTP redirect, returns +false+
+    # otherwise.
+    #
+    def redirect?
+      (300..307).include?(@code)
+    end
+
+    #
+    # Returns +true+ if the page was not found (returned 404 code),
+    # returns +false+ otherwise.
+    #
+    def not_found?
+      404 == @code
+    end
+
+    #
+    # Base URI from the HTML doc head element
+    # http://www.w3.org/TR/html4/struct/links.html#edef-BASE
+    #
+    def base
+      @base = if doc
+        href = doc.search('//head/base/@href')
+        URI(href.to_s) unless href.nil? rescue nil
+      end unless @base
+      
+      return nil if @base && @base.to_s().empty?
+      @base
+    end
+
+    #
+    # Converts relative URL *link* into an absolute URL based on the
+    # location of the page
+    #
+    def to_absolute(link)
+      return nil if link.nil?
+
+      # remove anchor
+      link = URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/,'')))
+
+      relative = URI(link)
+      absolute = base ? base.merge(relative) : @url.merge(relative)
+
+      absolute.path = '/' if absolute.path.empty?
+
+      return absolute
+    end
+
+    #
+    # Returns +true+ if *uri* is in the same domain as the page, returns
+    # +false+ otherwise
+    #
+    def in_domain?(uri)
+      @domain_aliases ||= []
+      uri.host == @url.host || @domain_aliases.include?(uri.host)
+    end
+
+    def to_hash
+      {'url' => @url.to_s,
+       'headers' => Marshal.dump(@headers),
+       'body' => @body,
+       'links' => links.map(&:to_s), 
+       'code' => @code,
+       'depth' => @depth,
+       'referer' => @referer.to_s,
+       'redirect_to' => @redirect_to.to_s,
+       'response_time' => @response_time,
+       'fetched' => @fetched,
+       'user_data' => @user_data.nil? ? {} : @user_data.marshal_dump
+     }
+    end
+
+    def to_json
+      th = to_hash.dup
+      th.each {|k,v| th.delete(k) if v.nil? || (v.respond_to?(:empty?) && v.empty?)}
+      th.delete('headers') if content_type.empty?
+      th.to_json
+    end
+
+    def self.from_hash(hash)
+      page = self.new(URI(hash['url']))
+      {'@headers' => hash['headers'] ? Marshal.load(hash['headers']) : {'content-type' => ['']},
+       '@body'    => hash['body'],
+       '@links'   => hash['links'] ? hash['links'].map { |link| URI(link) } : [],
+       '@code'    => hash['code'].to_i,
+       '@depth'   => hash['depth'].to_i,
+       '@referer' => hash['referer'],
+       '@redirect_to' => (!!hash['redirect_to'] && !hash['redirect_to'].empty?) ? URI(hash['redirect_to']) : nil,
+       '@response_time' => hash['response_time'].to_i,
+       '@fetched' => hash['fetched'],
+       '@user_data' => hash['user_data'] ? OpenStruct.new(hash['user_data']) : nil
+      }.each do |var, value|
+        page.instance_variable_set(var, value)
+      end
+      page
+    end
+
+    def self.from_json(json)
+      hash = JSON.parse json
+      self.from_hash hash
+    end
+  end
+end

data/lib/polipus/plugin.rb

@@ -0,0 +1,13 @@
+module Polipus
+  module Plugin
+    @@plugins = {}
+    def self.register plugin, options = {}
+      o = plugin.new(options)
+      @@plugins[o.class.name] = o
+    end
+
+    def self.plugins
+      @@plugins
+    end
+  end
+end

data/lib/polipus/plugins/cleaner.rb

@@ -0,0 +1,25 @@
+module Polipus
+  module Plugin
+    class Cleaner
+      
+      def initialize(options = {})
+        @reset = options[:reset] ||= false
+      end
+
+      def on_initialize crawler
+        crawler.logger.info {"Cleaner plugin loaded"}
+        unless @reset
+          crawler.logger.info {"Cleaner plugin is disable, add :reset => true to the plugin if you really know what you are doing"}
+          return nil
+        end
+        crawler.logger.info {"Cleaning all: url_tracker, storage, queue"}
+        Proc.new {
+          url_tracker.clear
+          storage.clear
+          queue_factory.clear
+          @options[:queue_overflow_adapter].clear if @options[:queue_overflow_adapter]
+        }
+      end
+    end
+  end
+end