polipus 0.3.0 → 0.3.1

data/lib/polipus/http.rb

@@ -1,6 +1,6 @@
-require "net/https"
-require "polipus/page"
-require "zlib"
+require 'net/https'
+require 'polipus/page'
+require 'zlib'
 require 'http/cookie'
 
 module Polipus
@@ -20,7 +20,8 @@ module Polipus
       Net::ProtocolError,
       SocketError,
       Timeout::Error,
-      Zlib::DataError
+      Zlib::DataError,
+      Zlib::GzipFile::Error
     ]
 
     def initialize(opts = {})
@@ -46,14 +47,14 @@ module Polipus
       pages = []
       get(url, referer) do |response, code, location, redirect_to, response_time|
         handle_compression response
-        pages << Page.new(location, :body          => response.body,
-                                    :code          => code,
-                                    :headers       => response.to_hash,
-                                    :referer       => referer,
-                                    :depth         => depth,
-                                    :redirect_to   => redirect_to,
-                                    :response_time => response_time,
-                                    :fetched_at    => Time.now.to_i)
+        pages << Page.new(location, body: response.body,
+                                    code: code,
+                                    headers: response.to_hash,
+                                    referer: referer,
+                                    depth: depth,
+                                    redirect_to: redirect_to,
+                                    response_time: response_time,
+                                    fetched_at: Time.now.to_i)
       end
 
       pages
@@ -81,7 +82,6 @@ module Polipus
       @opts[:user_agent]
     end
 
-   
     #
     # The proxy address string
     #
@@ -141,17 +141,18 @@ module Polipus
     def get(url, referer = nil)
       limit = redirect_limit
       loc = url
-      begin
-          # if redirected to a relative url, merge it with the host of the original
-          # request url
-          loc = url.merge(loc) if loc.relative?
-
-          response, response_time = get_response(loc, referer)
-          code = Integer(response.code)
-          redirect_to = response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
-          yield response, code, loc, redirect_to, response_time
-          limit -= 1
-      end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
+      loop do
+        # if redirected to a relative url, merge it with the host of the original
+        # request url
+        loc = url.merge(loc) if loc.relative?
+
+        response, response_time = get_response(loc, referer)
+        code = Integer(response.code)
+        redirect_to = response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
+        yield response, code, loc, redirect_to, response_time
+        limit -= 1
+        break unless (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
+      end
     end
 
     #
@@ -166,25 +167,24 @@ module Polipus
       opts['Cookie']  = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
       opts['Accept-Encoding'] = 'gzip,deflate'
 
-
       retries = 0
       begin
-        start = Time.now()
+        start = Time.now
         # format request
         req = Net::HTTP::Get.new(full_path, opts)
         # HTTP Basic authentication
         req.basic_auth url.user, url.password if url.user
         response = connection(url).request(req)
-        finish = Time.now()
+        finish = Time.now
         response_time = ((finish - start) * 1000).round
-        cookie_jar.parse(response["Set-Cookie"], url) if accept_cookies?
+        cookie_jar.parse(response['Set-Cookie'], url) if accept_cookies?
         return response, response_time
       rescue *RESCUABLE_ERRORS => e
         puts e.inspect if verbose?
         refresh_connection(url)
         retries += 1
-        unless retries > 3
-          retry 
+        if retries < 3
+          retry
         else
           raise e
         end
@@ -195,13 +195,13 @@ module Polipus
       @connections[url.host] ||= {}
       @connections_hits[url.host] ||= {}
 
-      if conn = @connections[url.host][url.port]
+      if @connections[url.host][url.port]
         if @opts[:connection_max_hits] && @connections_hits[url.host][url.port] >= @opts[:connection_max_hits]
-          @opts[:logger].debug {"Connection #{url.host}:#{url.port} is staled, refreshing"} if @opts[:logger]
+          @opts[:logger].debug { "Connection #{url.host}:#{url.port} is staled, refreshing" } if @opts[:logger]
           return refresh_connection url
         end
         @connections_hits[url.host][url.port] += 1
-        return conn
+        return @connections[url.host][url.port]
       end
 
       refresh_connection url
@@ -211,20 +211,20 @@ module Polipus
       proxy_host, proxy_port = proxy_host_port unless @opts[:proxy_host_port].nil?
 
       if @opts[:logger] && proxy_host && proxy_port
-        @opts[:logger].debug {"Request #{url} using proxy: #{proxy_host}:#{proxy_port}"}
+        @opts[:logger].debug { "Request #{url} using proxy: #{proxy_host}:#{proxy_port}" }
       end
 
       http = Net::HTTP.new(url.host, url.port, proxy_host, proxy_port)
 
-      http.read_timeout = read_timeout if !!read_timeout
-      http.open_timeout = open_timeout if !!open_timeout
+      http.read_timeout = read_timeout if read_timeout
+      http.open_timeout = open_timeout if open_timeout
 
       if url.scheme == 'https'
         http.use_ssl = true
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
       @connections_hits[url.host][url.port] = 1
-      @connections[url.host][url.port] = http.start 
+      @connections[url.host][url.port] = http.start
     end
 
     def verbose?
@@ -238,15 +238,14 @@ module Polipus
       to_url.host.nil? || (to_url.host == from_url.host)
     end
 
-    def handle_compression response
-      case response["content-encoding"]
-      when "gzip", "x-gzip"
+    def handle_compression(response)
+      case response['content-encoding']
+      when 'gzip', 'x-gzip'
         body_io = StringIO.new(response.body)
         response.body.replace Zlib::GzipReader.new(body_io).read
-      when "deflate"
+      when 'deflate'
         response.body.replace Zlib::Inflate.inflate(response.body)
       end
     end
-
   end
 end

data/lib/polipus/page.rb

@@ -4,7 +4,6 @@ require 'ostruct'
 require 'set'
 module Polipus
   class Page
-
     # The URL of the page
     attr_reader :url
     # The raw HTTP response body of the page
@@ -31,7 +30,7 @@ module Polipus
     attr_accessor :domain_aliases
 
     # Whether the current page should be stored
-    # Default: true  
+    # Default: true
     attr_accessor :storable
 
     attr_accessor :fetched_at
@@ -64,11 +63,11 @@ module Polipus
     def links
       return @links.to_a unless @links.nil?
       @links = Set.new
-      return [] if !doc
-      
-      doc.search("//a[@href]").each do |a|
+      return [] unless doc
+
+      doc.search('//a[@href]').each do |a|
         u = a['href']
-        next if u.nil? or u.empty?
+        next if u.nil? || u.empty?
         abs = to_absolute(u) rescue next
         @links << abs if in_domain?(abs)
       end
@@ -118,7 +117,7 @@ module Polipus
     # otherwise.
     #
     def html?
-      !!(content_type =~ %r{^(text/html|application/xhtml+xml)\b})
+      content_type =~ %r{^(text/html|application/xhtml+xml)\b}
     end
 
     #
@@ -151,11 +150,11 @@ module Polipus
     #
     def base
       @base = if doc
-        href = doc.search('//head/base/@href')
-        URI(href.to_s) unless href.nil? rescue nil
-      end unless @base
-      
-      return nil if @base && @base.to_s().empty?
+                href = doc.search('//head/base/@href')
+                URI(href.to_s) unless href.nil? rescue nil
+              end unless @base
+
+      return nil if @base && @base.to_s.empty?
       @base
     end
 
@@ -167,14 +166,14 @@ module Polipus
       return nil if link.nil?
 
       # remove anchor
-      link = URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/,'')))
+      link = URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/, '')))
 
       relative = URI(link)
       absolute = base ? base.merge(relative) : @url.merge(relative)
 
       absolute.path = '/' if absolute.path.empty?
 
-      return absolute
+      absolute
     end
 
     #
@@ -188,25 +187,25 @@ module Polipus
 
     def to_hash
       {
-       'url'           => @url.to_s,
-       'headers'       => Marshal.dump(@headers),
-       'body'          => @body,
-       'links'         => links.map(&:to_s), 
-       'code'          => @code,
-       'depth'         => @depth,
-       'referer'       => @referer.to_s,
-       'redirect_to'   => @redirect_to.to_s,
-       'response_time' => @response_time,
-       'fetched'       => @fetched,
-       'user_data'     => @user_data.nil? ? {} : @user_data.marshal_dump,
-       'fetched_at'    => @fetched_at,
-       'error'         => @error
-     }
+        'url'           => @url.to_s,
+        'headers'       => Marshal.dump(@headers),
+        'body'          => @body,
+        'links'         => links.map(&:to_s),
+        'code'          => @code,
+        'depth'         => @depth,
+        'referer'       => @referer.to_s,
+        'redirect_to'   => @redirect_to.to_s,
+        'response_time' => @response_time,
+        'fetched'       => @fetched,
+        'user_data'     => @user_data.nil? ? {} : @user_data.marshal_dump,
+        'fetched_at'    => @fetched_at,
+        'error'         => @error
+      }
     end
 
     def to_json
       th = to_hash.dup
-      th.each {|k,v| th.delete(k) if v.nil? || (v.respond_to?(:empty?) && v.empty?)}
+      th.each { |k, v| th.delete(k) if v.nil? || (v.respond_to?(:empty?) && v.empty?) }
       th.delete('headers') if content_type.empty?
       th.to_json
     end
@@ -220,21 +219,21 @@ module Polipus
       @storable
     end
 
-    def expired? ttl
+    def expired?(ttl)
       return false if fetched_at.nil?
       (Time.now.to_i - ttl) > fetched_at
     end
 
     def self.from_hash(hash)
-      page = self.new(URI(hash['url']))
+      page = new(URI(hash['url']))
       {
-        '@headers'       => hash['headers'] ? Marshal.load(hash['headers']) : {'content-type' => ['']},
+        '@headers'       => hash['headers'] ? Marshal.load(hash['headers']) : { 'content-type' => [''] },
         '@body'          => hash['body'],
         '@links'         => hash['links'] ? hash['links'].map { |link| URI(link) } : [],
         '@code'          => hash['code'].to_i,
         '@depth'         => hash['depth'].to_i,
         '@referer'       => hash['referer'],
-        '@redirect_to'   => (!!hash['redirect_to'] && !hash['redirect_to'].empty?) ? URI(hash['redirect_to']) : nil,
+        '@redirect_to'   => (hash['redirect_to'] && !hash['redirect_to'].empty?) ? URI(hash['redirect_to']) : nil,
         '@response_time' => hash['response_time'].to_i,
         '@fetched'       => hash['fetched'],
         '@user_data'     => hash['user_data'] ? OpenStruct.new(hash['user_data']) : nil,
@@ -248,7 +247,7 @@ module Polipus
 
     def self.from_json(json)
       hash = JSON.parse json
-      self.from_hash hash
+      from_hash hash
     end
   end
 end

data/lib/polipus/plugin.rb

@@ -1,7 +1,7 @@
 module Polipus
   module Plugin
     @@plugins = {}
-    def self.register plugin, options = {}
+    def self.register(plugin, options = {})
       o = plugin.new(options)
       @@plugins[o.class.name] = o
     end
@@ -10,4 +10,4 @@ module Polipus
       @@plugins
     end
   end
-end
+end

data/lib/polipus/plugins/cleaner.rb

@@ -1,25 +1,24 @@
 module Polipus
   module Plugin
     class Cleaner
-      
       def initialize(options = {})
         @reset = options[:reset] ||= false
       end
 
-      def on_initialize crawler
-        crawler.logger.info {"Cleaner plugin loaded"}
+      def on_initialize(crawler)
+        crawler.logger.info { 'Cleaner plugin loaded' }
         unless @reset
-          crawler.logger.info {"Cleaner plugin is disable, add :reset => true to the plugin if you really know what you are doing"}
+          crawler.logger.info { 'Cleaner plugin is disable, add :reset => true to the plugin if you really know what you are doing' }
           return nil
         end
-        crawler.logger.info {"Cleaning all: url_tracker, storage, queue"}
-        Proc.new {
+        crawler.logger.info { 'Cleaning all: url_tracker, storage, queue' }
+        proc do
           url_tracker.clear
           storage.clear
           queue_factory.clear
           @options[:queue_overflow_adapter].clear if @options[:queue_overflow_adapter]
-        }
+        end
       end
     end
   end
-end
+end

data/lib/polipus/plugins/sample.rb

@@ -1,17 +1,14 @@
 module Polipus
   module Plugin
     class Sample
-      
-      def initialize(options = {})
-        
+      def initialize(_options = {})
       end
 
-      def on_initialize crawler
-        Proc.new {
-          @options.each { |k,v| @logger.info {"Polipus configuration: #{k.to_s} => #{v}"} }
-        }
+      def on_initialize(_crawler)
+        proc do
+          @options.each { |k, v| @logger.info { "Polipus configuration: #{k} => #{v}" } }
+        end
       end
-      
     end
   end
-end
+end

data/lib/polipus/plugins/sleeper.rb

@@ -1,22 +1,21 @@
 module Polipus
   module Plugin
     class Sleeper
-      
       def initialize(options = {})
         @delay = options[:delay] ||= 1
       end
 
-      def on_initialize crawler
-        crawler.logger.info {"Sleeper plugin loaded, sleep for #{@delay} after each request"}
-        Proc.new {
+      def on_initialize(crawler)
+        crawler.logger.info { "Sleeper plugin loaded, sleep for #{@delay} after each request" }
+        proc do
           # Set to 1 the number of threads
           @options[:workers] = 1
-        }
+        end
       end
-      
-      def on_message_processed crawler
+
+      def on_message_processed(_crawler)
         sleep @delay
       end
     end
   end
-end
+end

data/lib/polipus/queue_overflow.rb

@@ -1,24 +1,24 @@
-require "polipus/queue_overflow/manager"
+require 'polipus/queue_overflow/manager'
 module Polipus
   module QueueOverflow
     def self.mongo_queue(mongo_db, queue_name, options = {})
-      require "polipus/queue_overflow/mongo_queue"
-      mongo_db ||= Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('polipus')
-      raise "First argument must be an instance of Mongo::DB" unless mongo_db.is_a?(Mongo::DB)
+      require 'polipus/queue_overflow/mongo_queue'
+      mongo_db ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
+      fail 'First argument must be an instance of Mongo::DB' unless mongo_db.is_a?(Mongo::DB)
       self::MongoQueue.new mongo_db, queue_name, options
     end
 
     def self.mongo_queue_capped(mongo_db, queue_name, options = {})
-      require "polipus/queue_overflow/mongo_queue_capped"
-      mongo_db ||= Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('polipus')
-      raise "First argument must be an instance of Mongo::DB" unless mongo_db.is_a?(Mongo::DB)
+      require 'polipus/queue_overflow/mongo_queue_capped'
+      mongo_db ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
+      fail 'First argument must be an instance of Mongo::DB' unless mongo_db.is_a?(Mongo::DB)
       options[:max] = 1_000_000 if options[:max].nil?
       self::MongoQueueCapped.new mongo_db, queue_name, options
     end
-    
-    def self.dev_null_queue(options = {})
-      require "polipus/queue_overflow/dev_null_queue"
+
+    def self.dev_null_queue(_options = {})
+      require 'polipus/queue_overflow/dev_null_queue'
       self::DevNullQueue.new
     end
   end
-end
+end