polipus 0.3.0 → 0.3.1

data/lib/polipus/queue_overflow/base.rb

@@ -3,4 +3,4 @@ module Polipus
     class Base
     end
   end
-end
+end

data/lib/polipus/queue_overflow/dev_null_queue.rb

@@ -1,4 +1,4 @@
-require "thread"
+require 'thread'
 module Polipus
   module QueueOverflow
     class DevNullQueue
@@ -16,18 +16,18 @@ module Polipus
       def clear
       end
 
-      def push data
+      def push(_data)
       end
 
       def pop(_ = false)
         nil
       end
-      
-      alias :size  :length
-      alias :dec   :pop
-      alias :shift :pop
-      alias :enc   :push
-      alias :<<    :push
+
+      alias_method :size,  :length
+      alias_method :dec,   :pop
+      alias_method :shift, :pop
+      alias_method :enc,   :push
+      alias_method :<<,    :push
     end
   end
-end
+end

data/lib/polipus/queue_overflow/manager.rb

@@ -10,43 +10,46 @@ module Polipus
         @redis = @polipus.redis
       end
 
-      def url_filter &block
+      def url_filter(&block)
         @url_filter = block
       end
 
       def perform
-        removed = 0
+        removed  = 0
         restored = 0
-
-        if @main_q.size > @item_limit
-          removed = rotate @main_q, @adapter,(@main_q.size - @item_limit)
-        elsif @main_q.size < @item_limit && !@adapter.empty?
-          restored = rotate @adapter, @main_q,(@item_limit -  @main_q.size)
+        main_q_size = @main_q.size
+        if main_q_size > @item_limit
+          @polipus.logger.info { "Overflow Manager: Going to offload items from redis: ~#{main_q_size - @item_limit}" }
+          removed = rotate(@main_q, @adapter) { @main_q.size > @item_limit }
+        elsif main_q_size < @item_limit && !@adapter.empty?
+          @polipus.logger.info { "Overflow Manager: Going to restore items into redis: ~#{@item_limit - main_q_size}" }
+          restored = rotate(@adapter, @main_q) { @main_q.size <= @item_limit }
         end
         [removed, restored]
       end
 
       private
-        def rotate source, dest, items
-          performed = 0
-          1.upto(items){|i|
-            message = source.pop(true)
-            if message
-              page = Page.from_json message
-              unless @polipus.storage.exists?(page)
-                allowed = !@url_filter.nil? ? @url_filter.call(page) : true
-                if allowed
-                  dest << message
-                  performed += 1
-                end
+
+      def rotate(source, dest)
+        performed = 0
+        loop do
+          message = source.pop(true)
+          if message
+            page = Page.from_json message
+            unless @polipus.storage.exists?(page)
+              allowed = @url_filter.nil? ? true : @url_filter.call(page)
+              if allowed
+                dest << message
+                performed += 1
               end
             end
-            source.commit if source.respond_to? :commit
-            @redis.expire "polipus_queue_overflow-#{@polipus.job_name}.lock", 180
-            break if !message || source.empty?
-          }
-          performed
+          end
+          source.commit if source.respond_to? :commit
+          break if !message || source.empty?
+          break unless yield source, dest
         end
+        performed
+      end
     end
   end
-end
+end

data/lib/polipus/queue_overflow/mongo_queue.rb

@@ -1,4 +1,5 @@
-require "thread"
+require 'thread'
+require 'mongo'
 module Polipus
   module QueueOverflow
     class MongoQueue
@@ -8,9 +9,7 @@ module Polipus
         @semaphore = Mutex.new
         @options = options
         @options[:ensure_uniq] ||= false
-        if @options[:ensure_uniq]
-          ensure_index
-        end
+        @options[:ensure_uniq] && ensure_index
       end
 
       def length
@@ -23,39 +22,38 @@ module Polipus
 
       def clear
         @mongo_db[@collection_name].drop
-        if @options[:ensure_uniq]
-          ensure_index
-        end
+        @options[:ensure_uniq] && ensure_index
       end
 
-      def push data
-        unless @options[:ensure_uniq]
-          @mongo_db[@collection_name].insert({:payload => data})  
+      def push(data)
+        if @options[:ensure_uniq]
+          @mongo_db[@collection_name].update({ payload: data }, { payload: data }, { upsert: 1, w: 1 })
         else
-          @mongo_db[@collection_name].update({:payload => data}, {:payload => data}, {:upsert => 1, :w => 1})
+          @mongo_db[@collection_name].insert(payload: data)
         end
-        true        
+        true
       end
 
       def pop(_ = false)
-        @semaphore.synchronize {
-          doc = @mongo_db[@collection_name].find({},:sort => {:_id => 1}).limit(1).first
+        @semaphore.synchronize do
+          doc = @mongo_db[@collection_name].find({}, sort: { _id: 1 }).limit(1).first
           return nil if doc.nil?
-          @mongo_db[@collection_name].remove(:_id => doc['_id'])
+          @mongo_db[@collection_name].remove(_id: doc['_id'])
           doc && doc['payload'] ? doc['payload'] : nil
-        }
+        end
       end
-      
-      alias :size  :length
-      alias :dec   :pop
-      alias :shift :pop
-      alias :enc   :push
-      alias :<<    :push
+
+      alias_method :size,  :length
+      alias_method :dec,   :pop
+      alias_method :shift, :pop
+      alias_method :enc,   :push
+      alias_method :<<,    :push
 
       protected
-        def ensure_index
-          @mongo_db[@collection_name].ensure_index({:payload => 1},{:background => 1, :unique => 1, :drop_dups => 1})
-        end
+
+      def ensure_index
+        @mongo_db[@collection_name].ensure_index({ payload: 1 }, { background: 1, unique: 1, drop_dups: 1 })
+      end
     end
   end
-end
+end

data/lib/polipus/queue_overflow/mongo_queue_capped.rb

@@ -1,4 +1,4 @@
-require "polipus/queue_overflow/mongo_queue"
+require 'polipus/queue_overflow/mongo_queue'
 module Polipus
   module QueueOverflow
     class MongoQueueCapped < MongoQueue
@@ -7,22 +7,22 @@ module Polipus
         @max = @options[:max]
       end
 
-      def push data
+      def push(data)
         super
-        @semaphore.synchronize {
+        @semaphore.synchronize do
           s = size
           if s > @max
-            docs = @mongo_db[@collection_name].find({},{:sort => {:_id => 1}, :fields => [:_id]}).limit(s-@max).map { |e| e['_id'] }
-            @mongo_db[@collection_name].remove({:_id => {'$in' => docs}, '$isolated' => 1})
+            docs = @mongo_db[@collection_name].find({}, { sort: { _id: 1 }, fields: [:_id] }).limit(s - @max).map { |e| e['_id'] }
+            @mongo_db[@collection_name].remove(:_id => { '$in' => docs }, '$isolated' => 1)
           end
-        }
+        end
       end
 
-      alias :size  :length
-      alias :dec   :pop
-      alias :shift :pop
-      alias :enc   :push
-      alias :<<    :push
+      alias_method :size,  :length
+      alias_method :dec,   :pop
+      alias_method :shift, :pop
+      alias_method :enc,   :push
+      alias_method :<<,    :push
     end
   end
-end
+end

data/lib/polipus/robotex.rb

@@ -2,22 +2,19 @@ require 'open-uri'
 require 'uri'
 require 'timeout'
 module Polipus
-
   # Original code taken from
   # https://github.com/chriskite/robotex/blob/master/lib/robotex.rb
 
   class Robotex
-    
     DEFAULT_TIMEOUT = 3
     VERSION = '1.0.0'
 
     attr_reader :user_agent
-    
+
     class ParsedRobots
-      
       def initialize(uri, user_agent)
         io = Robotex.get_robots_txt(uri, user_agent)
-        if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
+        if !io || io.content_type != 'text/plain' || io.status != %w(200 OK)
           io = StringIO.new("User-agent: *\nAllow: /\n")
         end
 
@@ -27,59 +24,55 @@ module Polipus
         agent = /.*/
         io.each do |line|
           next if line =~ /^\s*(#.*|$)/
-          arr = line.split(":")
+          arr = line.split(':')
           key = arr.shift
-          value = arr.join(":").strip
+          value = arr.join(':').strip
           value.strip!
           case key.downcase
-            when "user-agent"
-              agent = to_regex(value)
-            when "allow"
-              unless value.empty?
-                @allows[agent] ||= []
-                @allows[agent] << to_regex(value) 
-              end
-            when "disallow"
-              unless value.empty?
-                @disallows[agent] ||= []
-                @disallows[agent] << to_regex(value)
-              end
-            when "crawl-delay"
-              @delays[agent] = value.to_i
+          when 'user-agent'
+            agent = to_regex(value)
+          when 'allow'
+            unless value.empty?
+              @allows[agent] ||= []
+              @allows[agent] << to_regex(value)
+            end
+          when 'disallow'
+            unless value.empty?
+              @disallows[agent] ||= []
+              @disallows[agent] << to_regex(value)
+            end
+          when 'crawl-delay'
+            @delays[agent] = value.to_i
           end
         end
         @parsed = true
       end
-      
+
       def allowed?(uri, user_agent)
         return true unless @parsed
         allowed = true
         uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
         path = uri.request_uri
-        
+
         @allows.each do |key, value|
-          unless allowed      
+          unless allowed
             if user_agent =~ key
               value.each do |rule|
-                if path =~ rule
-                  allowed = true
-                end
+                path =~ rule && allowed = true
               end
             end
           end
         end
-        
+
         @disallows.each do |key, value|
           if user_agent =~ key
             value.each do |rule|
-              if path =~ rule
-                allowed = false
-              end
+              path =~ rule && allowed = false
             end
           end
         end
-        
-        return allowed
+
+        allowed
       end
 
       def delay(user_agent)
@@ -88,30 +81,28 @@ module Polipus
         end
         nil
       end
-      
+
       protected
-      
+
       def to_regex(pattern)
         pattern = Regexp.escape(pattern)
-        pattern.gsub!(Regexp.escape("*"), ".*")
+        pattern.gsub!(Regexp.escape('*'), '.*')
         Regexp.compile("^#{pattern}")
       end
     end
-    
+
     def self.get_robots_txt(uri, user_agent)
-      begin
-        Timeout::timeout(Robotex.timeout) do
-          URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
-        end 
-      rescue Timeout::Error
-        STDERR.puts "robots.txt request timed out"
+      Timeout.timeout(Robotex.timeout) do
+        URI.join(uri.to_s, '/robots.txt').open('User-Agent' => user_agent) rescue nil
       end
+    rescue Timeout::Error
+      STDERR.puts 'robots.txt request timed out'
     end
-    
-    def self.timeout=(t)
-      @timeout = t
+
+    class << self
+      attr_writer :timeout
     end
-    
+
     def self.timeout
       @timeout || DEFAULT_TIMEOUT
     end
@@ -127,7 +118,7 @@ module Polipus
       uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
       @parsed[uri.host] ||= ParsedRobots.new(uri, @user_agent)
     end
-    
+
     #
     # Download the server's robots.txt, and return try if we are allowed to acces the url, false otherwise
     #
@@ -146,9 +137,8 @@ module Polipus
     #
     def delay!(uri)
       delay = delay(uri)
-      sleep delay - (Time.now - @last_accessed) if !!delay
+      sleep delay - (Time.now - @last_accessed) if delay
       @last_accessed = Time.now
     end
-    
   end
-end
+end

data/lib/polipus/signal_handler.rb

@@ -0,0 +1,41 @@
+require 'singleton'
+module Polipus
+  class SignalHandler
+    include Singleton
+    attr_accessor :terminated
+    attr_accessor :enabled
+
+    def initialize
+      self.terminated = false
+      self.enabled = false
+    end
+
+    def self.enable
+      trap(:INT)  do
+        exit unless self.enabled?
+        terminate
+      end
+      trap(:TERM) do
+        exit unless self.enabled?
+        terminate
+      end
+      instance.enabled = true
+    end
+
+    def self.disable
+      instance.enabled = false
+    end
+
+    def self.terminate
+      instance.terminated = true
+    end
+
+    def self.terminated?
+      instance.terminated
+    end
+
+    def self.enabled?
+      instance.enabled
+    end
+  end
+end

data/lib/polipus/storage.rb

@@ -1,25 +1,25 @@
-require "polipus/storage/base"
+require 'polipus/storage/base'
 module Polipus
   module Storage
     def self.mongo_store(mongo = nil, collection_name = 'pages', except = [])
       require 'polipus/storage/mongo_store'
-      mongo ||= Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('polipus')
-      raise "First argument must be an instance of Mongo::DB" unless mongo.is_a?(Mongo::DB)
-      self::MongoStore.new(:mongo => mongo, :collection => collection_name, :except => except)
+      mongo ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
+      fail 'First argument must be an instance of Mongo::DB' unless mongo.is_a?(Mongo::DB)
+      self::MongoStore.new(mongo: mongo, collection: collection_name, except: except)
     end
 
     def self.s3_store(bucket_name = 'pages', aws_credential = {}, except = [])
       require 'polipus/storage/s3_store'
-      
+
       if aws_credential[:access_key_id].nil? || aws_credential[:secret_access_key].nil?
-        raise "You have to specify AWS crediantials: :access_key_id and :secret_access_key"
+        fail 'You have to specify AWS crediantials: :access_key_id and :secret_access_key'
       end
 
       self::S3Store.new(
-        :bucket             => bucket_name, 
-        :access_key_id      => aws_credential[:access_key_id], 
-        :secret_access_key  => aws_credential[:secret_access_key],
-        :except => except
+        bucket: bucket_name,
+        access_key_id: aws_credential[:access_key_id],
+        secret_access_key: aws_credential[:secret_access_key],
+        except: except
       )
     end
 
@@ -33,4 +33,4 @@ module Polipus
       self::MemoryStore.new
     end
   end
-end
+end