parallel588_polipus 0.4.0

data/lib/polipus/plugin.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+module Polipus
+  module Plugin
+    @@plugins = {}
+    def self.register(plugin, options = {})
+      o = plugin.new(options)
+      @@plugins[o.class.name] = o
+    end
+
+    def self.plugins
+      @@plugins
+    end
+  end
+end

data/lib/polipus/plugins/cleaner.rb

@@ -0,0 +1,25 @@
+# encoding: UTF-8
+module Polipus
+  module Plugin
+    class Cleaner
+      def initialize(options = {})
+        @reset = options[:reset] ||= false
+      end
+
+      def on_initialize(crawler)
+        crawler.logger.info { 'Cleaner plugin loaded' }
+        unless @reset
+          crawler.logger.info { 'Cleaner plugin is disabled, add :reset => true to the plugin if you really know what you are doing' }
+          return nil
+        end
+        crawler.logger.info { 'Cleaning all: url_tracker, storage, queue' }
+        proc do
+          url_tracker.clear
+          storage.clear
+          queue_factory.clear
+          @options[:queue_overflow_adapter].clear if @options[:queue_overflow_adapter]
+        end
+      end
+    end
+  end
+end

data/lib/polipus/plugins/sample.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+module Polipus
+  module Plugin
+    class Sample
+      def initialize(_options = {})
+      end
+
+      def on_initialize(_crawler)
+        proc do
+          @options.each { |k, v| @logger.info { "Polipus configuration: #{k} => #{v}" } }
+        end
+      end
+    end
+  end
+end

data/lib/polipus/plugins/sleeper.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+module Polipus
+  module Plugin
+    class Sleeper
+      def initialize(options = {})
+        @delay = options[:delay] ||= 1
+      end
+
+      def on_initialize(crawler)
+        crawler.logger.info { "Sleeper plugin loaded, sleep for #{@delay} after each request" }
+        proc do
+          # Set to 1 the number of threads
+          @options[:workers] = 1
+        end
+      end
+
+      def on_message_processed(_crawler)
+        sleep @delay
+      end
+    end
+  end
+end

data/lib/polipus/queue_overflow.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+require 'polipus/queue_overflow/manager'
+require 'polipus/queue_overflow/worker'
+module Polipus
+  module QueueOverflow
+    def self.mongo_queue(mongo_db, queue_name, options = {})
+      require 'polipus/queue_overflow/mongo_queue'
+      mongo_db ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
+      fail 'First argument must be an instance of Mongo::DB' unless mongo_db.is_a?(Mongo::DB)
+      self::MongoQueue.new mongo_db, queue_name, options
+    end
+
+    def self.mongo_queue_capped(mongo_db, queue_name, options = {})
+      require 'polipus/queue_overflow/mongo_queue_capped'
+      mongo_db ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
+      fail 'First argument must be an instance of Mongo::DB' unless mongo_db.is_a?(Mongo::DB)
+      options[:max] = 1_000_000 if options[:max].nil?
+      self::MongoQueueCapped.new mongo_db, queue_name, options
+    end
+
+    def self.dev_null_queue(_options = {})
+      require 'polipus/queue_overflow/dev_null_queue'
+      self::DevNullQueue.new
+    end
+  end
+end

data/lib/polipus/queue_overflow/base.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+module Polipus
+  module QueueOverflow
+    class Base
+    end
+  end
+end

data/lib/polipus/queue_overflow/dev_null_queue.rb

@@ -0,0 +1,34 @@
+# encoding: UTF-8
+require 'thread'
+module Polipus
+  module QueueOverflow
+    class DevNullQueue
+      def initialize
+      end
+
+      def length
+        0
+      end
+
+      def empty?
+        true
+      end
+
+      def clear
+      end
+
+      def push(_data)
+      end
+
+      def pop(_ = false)
+        nil
+      end
+
+      alias_method :size,  :length
+      alias_method :dec,   :pop
+      alias_method :shift, :pop
+      alias_method :enc,   :push
+      alias_method :<<,    :push
+    end
+  end
+end

data/lib/polipus/queue_overflow/manager.rb

@@ -0,0 +1,57 @@
+# encoding: UTF-8
+module Polipus
+  module QueueOverflow
+    class Manager
+      attr_accessor :url_filter
+      attr_reader :polipus
+      def initialize(polipus, main_q, item_limit)
+        @polipus = polipus
+        @main_q  = main_q
+        @adapter = @polipus.queue_overflow_adapter
+        @item_limit = item_limit
+        @redis = @polipus.redis
+      end
+
+      def url_filter(&block)
+        @url_filter = block
+      end
+
+      def perform
+        removed  = 0
+        restored = 0
+        main_q_size = @main_q.size
+        if main_q_size > @item_limit
+          @polipus.logger.info { "Overflow Manager: Going to offload items from redis: ~#{main_q_size - @item_limit}" }
+          removed = rotate(@main_q, @adapter) { @main_q.size > @item_limit }
+        elsif main_q_size < @item_limit && !@adapter.empty?
+          @polipus.logger.info { "Overflow Manager: Going to restore items into redis: ~#{@item_limit - main_q_size}" }
+          restored = rotate(@adapter, @main_q) { @main_q.size <= @item_limit }
+        end
+        [removed, restored]
+      end
+
+      private
+
+      def rotate(source, dest)
+        performed = 0
+        loop do
+          message = source.pop(true)
+          if message
+            page = Page.from_json message
+            unless @polipus.storage.exists?(page)
+              allowed = @url_filter.nil? ? true : @url_filter.call(page)
+              if allowed
+                dest << message
+                performed += 1
+              end
+            end
+          end
+          source.commit if source.respond_to? :commit
+          break if !message || source.empty?
+          break unless yield source, dest
+        end
+        performed
+      end
+    end
+  end
+end

data/lib/polipus/queue_overflow/mongo_queue.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+require 'thread'
+require 'mongo'
+module Polipus
+  module QueueOverflow
+    class MongoQueue
+      def initialize(mongo_db, queue_name, options = {})
+        @mongo_db = mongo_db
+        @collection_name = "polipus_q_overflow_#{queue_name}"
+        @semaphore = Mutex.new
+        @options = options
+        @options[:ensure_uniq] ||= false
+        @options[:ensure_uniq] && ensure_index
+      end
+
+      def length
+        @mongo_db[@collection_name].count
+      end
+
+      def empty?
+        !(length > 0)
+      end
+
+      def clear
+        @mongo_db[@collection_name].drop
+        @options[:ensure_uniq] && ensure_index
+      end
+
+      def push(data)
+        if @options[:ensure_uniq]
+          @mongo_db[@collection_name].update({ payload: data }, { payload: data }, { upsert: true, w: 1 })
+        else
+          @mongo_db[@collection_name].insert(payload: data)
+        end
+        true
+      end
+
+      def pop(_ = false)
+        @semaphore.synchronize do
+          doc = @mongo_db[@collection_name].find({}, sort: { _id: 1 }).limit(1).first
+          return nil if doc.nil?
+          @mongo_db[@collection_name].remove(_id: doc['_id'])
+          doc && doc['payload'] ? doc['payload'] : nil
+        end
+      end
+
+      alias_method :size,  :length
+      alias_method :dec,   :pop
+      alias_method :shift, :pop
+      alias_method :enc,   :push
+      alias_method :<<,    :push
+
+      protected
+
+      def ensure_index
+        @mongo_db[@collection_name].ensure_index({ payload: 1 }, { background: 1, unique: 1, drop_dups: 1 })
+      end
+    end
+  end
+end

data/lib/polipus/queue_overflow/mongo_queue_capped.rb

@@ -0,0 +1,29 @@
+# encoding: UTF-8
+require 'polipus/queue_overflow/mongo_queue'
+module Polipus
+  module QueueOverflow
+    class MongoQueueCapped < MongoQueue
+      def initialize(mongo_db, queue_name, options = {})
+        super
+        @max = @options[:max]
+      end
+
+      def push(data)
+        super
+        @semaphore.synchronize do
+          s = size
+          if s > @max
+            docs = @mongo_db[@collection_name].find({}, { sort: { _id: 1 }, fields: [:_id] }).limit(s - @max).map { |e| e['_id'] }
+            @mongo_db[@collection_name].remove(:_id => { '$in' => docs }, '$isolated' => 1)
+          end
+        end
+      end
+
+      alias_method :size,  :length
+      alias_method :dec,   :pop
+      alias_method :shift, :pop
+      alias_method :enc,   :push
+      alias_method :<<,    :push
+    end
+  end
+end

data/lib/polipus/queue_overflow/worker.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+module Polipus
+  module QueueOverflow
+    class Worker
+      def initialize(manager)
+        @logger  = manager.polipus.logger
+        @delay   = manager.polipus.options[:queue_overflow_manager_check_time]
+        @adapter = manager.polipus.queue_overflow_adapter
+        @manager = manager
+      end
+
+      def run
+        @logger.info { 'Overflow::Worker::run' }
+        loop do
+          @logger.info { 'Overflow Manager: cycle started' }
+          removed, restored = @manager.perform
+          @logger.info { "Overflow Manager: items removed=#{removed}, items restored=#{restored}, items stored=#{@adapter.size}" }
+          sleep @delay
+          break if SignalHandler.terminated?
+        end
+      end
+    end
+  end
+end

data/lib/polipus/robotex.rb

@@ -0,0 +1,145 @@
+# encoding: UTF-8
+require 'open-uri'
+require 'uri'
+require 'timeout'
+module Polipus
+  # Original code taken from
+  # https://github.com/chriskite/robotex/blob/master/lib/robotex.rb
+
+  class Robotex
+    DEFAULT_TIMEOUT = 3
+    VERSION = '1.0.0'
+
+    attr_reader :user_agent
+
+    class ParsedRobots
+      def initialize(uri, user_agent)
+        io = Robotex.get_robots_txt(uri, user_agent)
+        if !io || io.content_type != 'text/plain' || io.status != %w(200 OK)
+          io = StringIO.new("User-agent: *\nAllow: /\n")
+        end
+
+        @disallows = {}
+        @allows = {}
+        @delays = {}
+        agent = /.*/
+        io.each do |line|
+          next if line =~ /^\s*(#.*|$)/
+          arr = line.split(':')
+          key = arr.shift
+          value = arr.join(':').strip
+          value.strip!
+          case key.downcase
+          when 'user-agent'
+            agent = to_regex(value)
+          when 'allow'
+            unless value.empty?
+              @allows[agent] ||= []
+              @allows[agent] << to_regex(value)
+            end
+          when 'disallow'
+            unless value.empty?
+              @disallows[agent] ||= []
+              @disallows[agent] << to_regex(value)
+            end
+          when 'crawl-delay'
+            @delays[agent] = value.to_i
+          end
+        end
+        @parsed = true
+      end
+
+      def allowed?(uri, user_agent)
+        return true unless @parsed
+        allowed = true
+        uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+        path = uri.request_uri
+
+        @allows.each do |key, value|
+          unless allowed
+            if user_agent =~ key
+              value.each do |rule|
+                path =~ rule && allowed = true
+              end
+            end
+          end
+        end
+
+        @disallows.each do |key, value|
+          if user_agent =~ key
+            value.each do |rule|
+              path =~ rule && allowed = false
+            end
+          end
+        end
+
+        allowed
+      end
+
+      def delay(user_agent)
+        @delays.each do |agent, delay|
+          return delay if agent =~ user_agent
+        end
+        nil
+      end
+
+      protected
+
+      def to_regex(pattern)
+        pattern = Regexp.escape(pattern)
+        pattern.gsub!(Regexp.escape('*'), '.*')
+        Regexp.compile("^#{pattern}")
+      end
+    end
+
+    def self.get_robots_txt(uri, user_agent)
+      Timeout.timeout(Robotex.timeout) do
+        URI.join(uri.to_s, '/robots.txt').open('User-Agent' => user_agent) rescue nil
+      end
+    rescue Timeout::Error
+      STDERR.puts 'robots.txt request timed out'
+    end
+
+    class << self
+      attr_writer :timeout
+    end
+
+    def self.timeout
+      @timeout || DEFAULT_TIMEOUT
+    end
+
+    def initialize(user_agent = nil)
+      user_agent = "Robotex/#{VERSION} (http://www.github.com/chriskite/robotex)" if user_agent.nil?
+      @user_agent = user_agent
+      @last_accessed = Time.at(1)
+      @parsed = {}
+    end
+
+    def parse_host(uri)
+      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+      @parsed[uri.host] ||= ParsedRobots.new(uri, @user_agent)
+    end
+
+    #
+    # Download the server's robots.txt, and return try if we are allowed to acces the url, false otherwise
+    #
+    def allowed?(uri)
+      parse_host(uri).allowed?(uri, @user_agent)
+    end
+
+    #
+    # Return the value of the Crawl-Delay directive, or nil if none
+    def delay(uri)
+      parse_host(uri).delay(@user_agent)
+    end
+
+    #
+    # Sleep for the amount of time necessary to obey the Crawl-Delay specified by the server
+    #
+    def delay!(uri)
+      delay = delay(uri)
+      sleep delay - (Time.now - @last_accessed) if delay
+      @last_accessed = Time.now
+    end
+  end
+end