polipus 0.0.1

data/lib/polipus/plugins/sample.rb

@@ -0,0 +1,17 @@
+module Polipus
+  module Plugin
+    class Sample
+      
+      def initialize(options = {})
+        
+      end
+
+      def on_initialize crawler
+        Proc.new {
+          @options.each { |k,v| @logger.info {"Polipus configuration: #{k.to_s} => #{v}"} }
+        }
+      end
+      
+    end
+  end
+end

data/lib/polipus/plugins/sleeper.rb

@@ -0,0 +1,22 @@
+module Polipus
+  module Plugin
+    class Sleeper
+      
+      def initialize(options = {})
+        @delay = options[:delay] ||= 1
+      end
+
+      def on_initialize crawler
+        crawler.logger.info {"Sleeper plugin loaded, sleep for #{@delay} after each request"}
+        Proc.new {
+          # Set to 1 the number of threads
+          @options[:workers] = 1
+        }
+      end
+      
+      def on_message_processed crawler
+        sleep @delay
+      end
+    end
+  end
+end

data/lib/polipus/queue_overflow.rb

@@ -0,0 +1,24 @@
+require "polipus/queue_overflow/manager"
+module Polipus
+  module QueueOverflow
+    def self.mongo_queue(mongo_db, queue_name, options = {})
+      require "polipus/queue_overflow/mongo_queue"
+      mongo_db ||= Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('polipus')
+      raise "First argument must be an instance of Mongo::DB" unless mongo_db.is_a?(Mongo::DB)
+      self::MongoQueue.new mongo_db, queue_name, options
+    end
+
+    def self.mongo_queue_capped(mongo_db, queue_name, options = {})
+      require "polipus/queue_overflow/mongo_queue_capped"
+      mongo_db ||= Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('polipus')
+      raise "First argument must be an instance of Mongo::DB" unless mongo_db.is_a?(Mongo::DB)
+      options[:max] = 1_000_000 if options[:max].nil?
+      self::MongoQueueCapped.new mongo_db, queue_name, options
+    end
+    
+    def self.dev_null_queue(options = {})
+      require "polipus/queue_overflow/dev_null_queue"
+      self::DevNullQueue.new
+    end
+  end
+end

data/lib/polipus/queue_overflow/base.rb

@@ -0,0 +1,6 @@
+module Polipus
+  module QueueOverflow
+    class Base
+    end
+  end
+end

data/lib/polipus/queue_overflow/dev_null_queue.rb

@@ -0,0 +1,33 @@
+require "thread"
+module Polipus
+  module QueueOverflow
+    class DevNullQueue
+      def initialize
+      end
+
+      def length
+        0
+      end
+
+      def empty?
+        true
+      end
+
+      def clear
+      end
+
+      def push data
+      end
+
+      def pop(_ = false)
+        nil
+      end
+      
+      alias :size  :length
+      alias :dec   :pop
+      alias :shift :pop
+      alias :enc   :push
+      alias :<<    :push
+    end
+  end
+end

data/lib/polipus/queue_overflow/manager.rb

@@ -0,0 +1,50 @@
+module Polipus
+  module QueueOverflow
+    class Manager
+      attr_accessor :url_filter
+      def initialize(polipus, main_q, item_limit)
+        @polipus = polipus
+        @main_q  = main_q
+        @adapter = @polipus.queue_overflow_adapter
+        @item_limit = item_limit
+      end
+
+      def url_filter &block
+        @url_filter = block
+      end
+
+      def perform
+        removed = 0
+        restored = 0
+
+        if @main_q.size > @item_limit
+          removed = rotate @main_q, @adapter,(@main_q.size - @item_limit)
+        elsif @main_q.size < @item_limit && !@adapter.empty?
+          restored = rotate @adapter, @main_q,(@item_limit -  @main_q.size)
+        end
+        [removed, restored]
+      end
+
+      private
+        def rotate source, dest, items
+          performed = 0
+          1.upto(items){|i|
+            message = source.pop(true)
+            if message
+              page = Page.from_json message
+              unless @polipus.storage.exists?(page)
+                allowed = !@url_filter.nil? ? @url_filter.call(page) : true
+                if allowed
+                  dest << message
+                  performed += 1
+                end
+              end
+            end
+            source.commit if source.respond_to? :commit
+            break if !message || source.empty?
+          }
+          performed
+        end
+    end
+  end
+end

data/lib/polipus/queue_overflow/mongo_queue.rb

@@ -0,0 +1,61 @@
+require "thread"
+module Polipus
+  module QueueOverflow
+    class MongoQueue
+      def initialize(mongo_db, queue_name, options = {})
+        @mongo_db = mongo_db
+        @collection_name = "polipus_q_overflow_#{queue_name}"
+        @semaphore = Mutex.new
+        @options = options
+        @options[:ensure_uniq] ||= false
+        if @options[:ensure_uniq]
+          ensure_index
+        end
+      end
+
+      def length
+        @mongo_db[@collection_name].count
+      end
+
+      def empty?
+        !(length > 0)
+      end
+
+      def clear
+        @mongo_db[@collection_name].drop
+        if @options[:ensure_uniq]
+          ensure_index
+        end
+      end
+
+      def push data
+        unless @options[:ensure_uniq]
+          @mongo_db[@collection_name].insert({:payload => data})  
+        else
+          @mongo_db[@collection_name].update({:payload => data}, {:payload => data}, {:upsert => 1, :w => 1})
+        end
+        true        
+      end
+
+      def pop(_ = false)
+        @semaphore.synchronize {
+          doc = @mongo_db[@collection_name].find({},:sort => {:_id => 1}).limit(1).first
+          return nil if doc.nil?
+          @mongo_db[@collection_name].remove(:_id => doc['_id'])
+          doc && doc['payload'] ? doc['payload'] : nil
+        }
+      end
+      
+      alias :size  :length
+      alias :dec   :pop
+      alias :shift :pop
+      alias :enc   :push
+      alias :<<    :push
+
+      protected
+        def ensure_index
+          @mongo_db[@collection_name].ensure_index({:payload => 1},{:background => 1, :unique => 1, :drop_dups => 1})
+        end
+    end
+  end
+end

data/lib/polipus/queue_overflow/mongo_queue_capped.rb

@@ -0,0 +1,28 @@
+require "polipus/queue_overflow/mongo_queue"
+module Polipus
+  module QueueOverflow
+    class MongoQueueCapped < MongoQueue
+      def initialize(mongo_db, queue_name, options = {})
+        super
+        @max = @options[:max]
+      end
+
+      def push data
+        super
+        @semaphore.synchronize {
+          s = size
+          if s > @max
+            docs = @mongo_db[@collection_name].find({},{:sort => {:_id => 1}, :fields => [:_id]}).limit(s-@max).map { |e| e['_id'] }
+            @mongo_db[@collection_name].remove({:_id => {'$in' => docs}, '$isolated' => 1})
+          end
+        }
+      end
+
+      alias :size  :length
+      alias :dec   :pop
+      alias :shift :pop
+      alias :enc   :push
+      alias :<<    :push
+    end
+  end
+end

data/lib/polipus/storage.rb

@@ -0,0 +1,31 @@
+require "polipus/storage/base"
+module Polipus
+  module Storage
+    def self.mongo_store(mongo = nil, collection_name = 'pages', except = [])
+      require 'polipus/storage/mongo_store'
+      mongo ||= Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('polipus')
+      raise "First argument must be an instance of Mongo::DB" unless mongo.is_a?(Mongo::DB)
+      self::MongoStore.new(:mongo => mongo, :collection => collection_name, :except => except)
+    end
+
+    def self.s3_store(bucket_name = 'pages', aws_credential = {}, except = [])
+      require 'polipus/storage/s3_store'
+      
+      if aws_credential[:access_key_id].nil? || aws_credential[:secret_access_key].nil?
+        raise "You have to specify AWS crediantials: :access_key_id and :secret_access_key"
+      end
+
+      self::S3Store.new(
+        :bucket             => bucket_name, 
+        :access_key_id      => aws_credential[:access_key_id], 
+        :secret_access_key  => aws_credential[:secret_access_key],
+        :except => except
+      )
+    end
+
+    def self.dev_null
+      require 'polipus/storage/dev_null'
+      self::DevNull.new
+    end
+  end
+end

data/lib/polipus/storage/base.rb

@@ -0,0 +1,17 @@
+require "uri"
+
+module Polipus
+  module Storage
+    class Base
+      attr_accessor :include_query_string_in_uuid
+      protected
+        def uuid page
+          if @include_query_string_in_uuid.nil?
+            @include_query_string_in_uuid = true
+          end
+          url_to_hash = @include_query_string_in_uuid ? page.url.to_s : page.url.to_s.gsub(/\?.*$/,'')
+          Digest::MD5.hexdigest(url_to_hash) 
+        end
+    end
+  end
+end

data/lib/polipus/storage/dev_null.rb

@@ -0,0 +1,35 @@
+module Polipus
+  module Storage
+    class DevNull < Base
+
+      def initialize(options = {})
+      end
+
+      def add page
+      end
+
+      def exists?(page)
+        false
+      end
+
+      def get page
+        nil
+      end
+
+      def remove page
+        false
+      end
+
+      def count
+        0
+      end
+
+      def each
+        yield nil
+      end
+
+      def clear
+      end
+    end
+  end
+end

data/lib/polipus/storage/mongo_store.rb

@@ -0,0 +1,86 @@
+require "mongo"
+require "zlib"
+require "thread"
+module Polipus
+  module Storage
+    class MongoStore < Base
+      BINARY_FIELDS = %w(body headers data)
+      def initialize(options = {})
+        @mongo      = options[:mongo]
+        @collection = options[:collection]
+        @mongo.create_collection(@collection)
+        @mongo[@collection].ensure_index(:uuid, :unique => true, :drop_dups => true, :background => true)
+        @compress_body = options[:compress_body] ||= true
+        @except = options[:except] ||= []
+        @semaphore = Mutex.new
+      end
+
+      def add page
+        @semaphore.synchronize {
+          obj = page.to_hash
+          @except.each {|e| obj.delete e.to_s}
+          obj['uuid'] = uuid(page)
+          obj['body'] = Zlib::Deflate.deflate(obj['body']) if @compress_body && obj['body']
+          BINARY_FIELDS.each do |field|
+            obj[field] = BSON::Binary.new(obj[field]) unless obj[field].nil?
+          end
+          @mongo[@collection].update({:uuid => obj['uuid']}, obj, {:upsert => true, :w => 1})
+          obj['uuid']
+        }
+      end
+
+      def exists?(page)
+        @semaphore.synchronize {
+          doc = @mongo[@collection].find({:uuid => uuid(page)}, {:fields => [:_id]}).limit(1).first
+          !doc.nil?
+        }
+      end
+
+      def get page
+        @semaphore.synchronize {
+          data = @mongo[@collection].find({:uuid => uuid(page)}).limit(1).first
+          if data
+            return load_page(data)
+          end
+        }
+      end
+
+      def remove page
+        @semaphore.synchronize {
+          @mongo[@collection].remove({:uuid => uuid(page)})
+        }
+      end
+
+      def count
+        @mongo[@collection].count
+      end
+
+      def each
+        @mongo[@collection].find({},:timeout => false) do |cursor|
+          cursor.each do |doc|
+            page = load_page(doc)
+            yield doc['uuid'], page 
+          end
+        end
+      end
+
+      def clear
+        @mongo[@collection].drop
+      end
+
+      private
+        def load_page(hash)
+          BINARY_FIELDS.each do |field|
+            hash[field] = hash[field].to_s
+          end
+          begin
+            hash['body'] = Zlib::Inflate.inflate(hash['body']) if @compress_body && hash['body'] && !hash['body'].empty?
+            return Page.from_hash(hash)
+          rescue
+          end
+          nil
+        end
+
+    end
+  end
+end

data/lib/polipus/storage/s3_store.rb

@@ -0,0 +1,100 @@
+require "aws/s3"
+require "zlib"
+require "thread"
+require "json"
+module Polipus
+  module Storage
+    class S3Store < Base
+      def initialize(options = {})
+        @options = options
+        @except = @options[:except] ||= []
+        @semaphore = Mutex.new
+
+        AWS::S3::Base.establish_connection!(
+          :access_key_id     => @options[:access_key_id],
+          :secret_access_key => @options[:secret_access_key]
+        )
+        @options[:bucket] = "com.polipus.pages.#{@options[:bucket]}"
+        begin
+          @bucket = AWS::S3::Bucket.find(@options[:bucket])
+        rescue AWS::S3::NoSuchBucket
+          create_bucket
+        end
+      end
+
+      def add page
+        @semaphore.synchronize {
+          obj = page.to_hash
+          @except.each {|e| obj.delete e.to_s}
+          puuid = uuid(page)
+          obj['uuid'] = puuid
+          data = Zlib::Deflate.deflate(obj.to_json)
+          AWS::S3::S3Object.store(puuid, data, @bucket.name)
+          puuid
+        }
+      end
+
+      def exists?(page)
+        AWS::S3::S3Object.exists? uuid(page), @bucket.name
+      end
+
+      def get page
+        @semaphore.synchronize {
+          if exists?(page)
+            data = AWS::S3::S3Object.find(uuid(page), @bucket.name).value
+            return load_page(data)
+          end
+          nil
+        }
+      end
+
+      def remove page
+        @semaphore.synchronize {
+          if exists?(page)
+            AWS::S3::S3Object.delete(uuid(page), @bucket.name)
+          end
+          true
+        }
+      end
+
+      def count
+        @bucket.size
+      end
+
+      def clear
+        AWS::S3::Bucket.delete(@bucket.name, :force => true)
+        create_bucket
+      end
+
+      def each
+        objects = []
+        last_key = nil
+        begin
+          objects = AWS::S3::Bucket.objects(@bucket.name, :marker => last_key)
+          break if objects.size == 0
+          objects.each do |o|
+            page = load_page(o.value)
+            yield o.key, page 
+          end
+          last_key   = objects.last.key
+        end while true
+      end
+
+      private
+        def load_page(data)
+          begin
+            payload = Zlib::Inflate.inflate(data)
+            hash = JSON.parse(payload)
+            return Page.from_hash(hash)
+          rescue
+          end
+          nil
+        end
+
+        def create_bucket
+          AWS::S3::Bucket.create(@options[:bucket])
+          @bucket = AWS::S3::Bucket.find(@options[:bucket])
+        end
+    end
+  end
+end