polipus 0.3.0 → 0.3.1

data/lib/polipus/storage/base.rb

@@ -1,17 +1,19 @@
-require "uri"
+require 'uri'
 
 module Polipus
   module Storage
     class Base
       attr_accessor :include_query_string_in_uuid
+
       protected
-        def uuid page
-          if @include_query_string_in_uuid.nil?
-            @include_query_string_in_uuid = true
-          end
-          url_to_hash = @include_query_string_in_uuid ? page.url.to_s : page.url.to_s.gsub(/\?.*$/,'')
-          Digest::MD5.hexdigest(url_to_hash) 
+
+      def uuid(page)
+        if @include_query_string_in_uuid.nil?
+          @include_query_string_in_uuid = true
         end
+        url_to_hash = @include_query_string_in_uuid ? page.url.to_s : page.url.to_s.gsub(/\?.*$/, '')
+        Digest::MD5.hexdigest(url_to_hash)
+      end
     end
   end
-end
+end

data/lib/polipus/storage/dev_null.rb

@@ -1,22 +1,21 @@
 module Polipus
   module Storage
     class DevNull < Base
-
-      def initialize(options = {})
+      def initialize(_options = {})
       end
 
-      def add page
+      def add(_page)
       end
 
-      def exists?(page)
+      def exists?(_page)
         false
       end
 
-      def get page
+      def get(_page)
         nil
       end
 
-      def remove page
+      def remove(_page)
         false
       end
 
@@ -32,4 +31,4 @@ module Polipus
       end
     end
   end
-end
+end

data/lib/polipus/storage/memory_store.rb

@@ -1,56 +1,55 @@
-require "thread"
+require 'thread'
 module Polipus
   module Storage
     class MemoryStore < Base
-
-      def initialize(options = {})
-        @store = Hash.new
+      def initialize(_options = {})
+        @store = {}
         @semaphore = Mutex.new
       end
 
-      def add page
-        @semaphore.synchronize {
+      def add(page)
+        @semaphore.synchronize do
           u = uuid(page)
           @store[u] = page
           u
-        }
+        end
       end
 
       def exists?(page)
-        @semaphore.synchronize {
+        @semaphore.synchronize do
           @store.key?(uuid(page))
-        }
+        end
       end
 
-      def get page
-        @semaphore.synchronize {
+      def get(page)
+        @semaphore.synchronize do
           @store[uuid(page)]
-        }
+        end
       end
 
-      def remove page
-        @semaphore.synchronize {
+      def remove(page)
+        @semaphore.synchronize do
           @store.delete(uuid(page))
-        }
+        end
       end
 
       def count
-        @semaphore.synchronize {
+        @semaphore.synchronize do
           @store.count
-        }
+        end
       end
 
       def each
-        @store.each do |k,v|
-          yield k,v
+        @store.each do |k, v|
+          yield k, v
         end
       end
 
       def clear
-        @semaphore.synchronize {
+        @semaphore.synchronize do
           @store = Hash.new
-        }
+        end
       end
     end
   end
-end
+end

data/lib/polipus/storage/mongo_store.rb

@@ -1,6 +1,6 @@
-require "mongo"
-require "zlib"
-require "thread"
+require 'mongo'
+require 'zlib'
+require 'thread'
 module Polipus
   module Storage
     class MongoStore < Base
@@ -9,44 +9,44 @@ module Polipus
         @mongo      = options[:mongo]
         @collection = options[:collection]
         @mongo.create_collection(@collection)
-        @mongo[@collection].ensure_index(:uuid, :unique => true, :drop_dups => true, :background => true)
+        @mongo[@collection].ensure_index(:uuid, unique: true, drop_dups: true, background: true)
         @compress_body = options[:compress_body] ||= true
         @except = options[:except] ||= []
         @semaphore = Mutex.new
       end
 
-      def add page
-        @semaphore.synchronize {
+      def add(page)
+        @semaphore.synchronize do
           obj = page.to_hash
-          @except.each {|e| obj.delete e.to_s}
+          @except.each { |e| obj.delete e.to_s }
           obj['uuid'] = uuid(page)
           obj['body'] = Zlib::Deflate.deflate(obj['body']) if @compress_body && obj['body']
           BINARY_FIELDS.each do |field|
             obj[field] = BSON::Binary.new(obj[field]) unless obj[field].nil?
           end
-          @mongo[@collection].update({:uuid => obj['uuid']}, obj, {:upsert => true, :w => 1})
+          @mongo[@collection].update({ uuid: obj['uuid'] }, obj, upsert: true, w: 1)
           obj['uuid']
-        }
+        end
       end
 
       def exists?(page)
-        @semaphore.synchronize {
-          doc = @mongo[@collection].find({:uuid => uuid(page)}, {:fields => [:_id]}).limit(1).first
+        @semaphore.synchronize do
+          doc = @mongo[@collection].find({ uuid: uuid(page) }, { fields: [:_id] }).limit(1).first
           !doc.nil?
-        }
+        end
       end
 
-      def get page
-        @semaphore.synchronize {
-          data = @mongo[@collection].find({:uuid => uuid(page)}).limit(1).first
+      def get(page)
+        @semaphore.synchronize do
+          data = @mongo[@collection].find(uuid: uuid(page)).limit(1).first
           return load_page(data) if data
-        }
+        end
       end
 
-      def remove page
-        @semaphore.synchronize {
-          @mongo[@collection].remove({:uuid => uuid(page)})
-        }
+      def remove(page)
+        @semaphore.synchronize do
+          @mongo[@collection].remove(uuid: uuid(page))
+        end
       end
 
       def count
@@ -54,10 +54,10 @@ module Polipus
       end
 
       def each
-        @mongo[@collection].find({},:timeout => false) do |cursor|
+        @mongo[@collection].find({}, timeout: false) do |cursor|
           cursor.each do |doc|
             page = load_page(doc)
-            yield doc['uuid'], page 
+            yield doc['uuid'], page
           end
         end
       end
@@ -67,22 +67,18 @@ module Polipus
       end
 
       private
-        def load_page(hash)
-          BINARY_FIELDS.each do |field|
-            hash[field] = hash[field].to_s
-          end
-          begin
-            hash['body'] = Zlib::Inflate.inflate(hash['body']) if @compress_body && hash['body'] && !hash['body'].empty?
-            page = Page.from_hash(hash)
-            if page.fetched_at.nil?
-              page.fetched_at = hash['_id'].generation_time.to_i
-            end
-            return page
-          rescue
-          end
-          nil
-        end
 
+      def load_page(hash)
+        BINARY_FIELDS.each do |field|
+          hash[field] = hash[field].to_s
+        end
+        hash['body'] = Zlib::Inflate.inflate(hash['body']) if @compress_body && hash['body'] && !hash['body'].empty?
+        page = Page.from_hash(hash)
+        if page.fetched_at.nil?
+          page.fetched_at = hash['_id'].generation_time.to_i
+        end
+        page
+      end
     end
   end
-end
+end

data/lib/polipus/storage/s3_store.rb

@@ -1,7 +1,7 @@
-require "aws/s3"
-require "zlib"
-require "thread"
-require "json"
+require 'aws/s3'
+require 'zlib'
+require 'thread'
+require 'json'
 module Polipus
   module Storage
     class S3Store < Base
@@ -11,8 +11,8 @@ module Polipus
         @semaphore = Mutex.new
 
         AWS::S3::Base.establish_connection!(
-          :access_key_id     => @options[:access_key_id],
-          :secret_access_key => @options[:secret_access_key]
+          access_key_id: @options[:access_key_id],
+          secret_access_key: @options[:secret_access_key]
         )
         @options[:bucket] = "com.polipus.pages.#{@options[:bucket]}"
         begin
@@ -22,39 +22,37 @@ module Polipus
         end
       end
 
-      def add page
-        @semaphore.synchronize {
+      def add(page)
+        @semaphore.synchronize do
           obj = page.to_hash
-          @except.each {|e| obj.delete e.to_s}
+          @except.each { |e| obj.delete e.to_s }
           puuid = uuid(page)
           obj['uuid'] = puuid
           data = Zlib::Deflate.deflate(obj.to_json)
           AWS::S3::S3Object.store(puuid, data, @bucket.name)
           puuid
-        }
+        end
       end
 
       def exists?(page)
         AWS::S3::S3Object.exists? uuid(page), @bucket.name
       end
 
-      def get page
-        @semaphore.synchronize {
+      def get(page)
+        @semaphore.synchronize do
           if exists?(page)
             data = AWS::S3::S3Object.find(uuid(page), @bucket.name).value
             return load_page(data)
           end
           nil
-        }
+        end
       end
 
-      def remove page
-        @semaphore.synchronize {
-          if exists?(page)
-            AWS::S3::S3Object.delete(uuid(page), @bucket.name)
-          end
+      def remove(page)
+        @semaphore.synchronize do
+          exists?(page) && AWS::S3::S3Object.delete(uuid(page), @bucket.name)
           true
-        }
+        end
       end
 
       def count
@@ -62,39 +60,36 @@ module Polipus
       end
 
       def clear
-        AWS::S3::Bucket.delete(@bucket.name, :force => true)
+        AWS::S3::Bucket.delete(@bucket.name, force: true)
         create_bucket
       end
 
       def each
         objects = []
         last_key = nil
-        begin
-          objects = AWS::S3::Bucket.objects(@bucket.name, :marker => last_key)
+        loop do
+          objects = AWS::S3::Bucket.objects(@bucket.name, marker: last_key)
           break if objects.size == 0
           objects.each do |o|
             page = load_page(o.value)
-            yield o.key, page 
+            yield o.key, page
           end
           last_key   = objects.last.key
-        end while true
+        end
       end
 
       private
-        def load_page(data)
-          begin
-            payload = Zlib::Inflate.inflate(data)
-            hash = JSON.parse(payload)
-            return Page.from_hash(hash)
-          rescue
-          end
-          nil
-        end
 
-        def create_bucket
-          AWS::S3::Bucket.create(@options[:bucket])
-          @bucket = AWS::S3::Bucket.find(@options[:bucket])
-        end
+      def load_page(data)
+        payload = Zlib::Inflate.inflate(data)
+        hash = JSON.parse(payload)
+        Page.from_hash(hash)
+      end
+
+      def create_bucket
+        AWS::S3::Bucket.create(@options[:bucket])
+        @bucket = AWS::S3::Bucket.find(@options[:bucket])
+      end
     end
   end
-end
+end

data/lib/polipus/url_tracker.rb

@@ -1,7 +1,7 @@
 module Polipus
   module UrlTracker
     def self.bloomfilter(options = {})
-      require "polipus/url_tracker/bloomfilter"
+      require 'polipus/url_tracker/bloomfilter'
       options[:size]       ||= 1_000_000
       options[:error_rate] ||= 0.01
       options[:key_name]   ||= 'polipus-bloomfilter'
@@ -11,10 +11,10 @@ module Polipus
     end
 
     def self.redis_set(options = {})
-      require "polipus/url_tracker/redis_set"
+      require 'polipus/url_tracker/redis_set'
       options[:redis]      ||= Redis.current
       options[:key_name]   ||= 'polipus-set'
       self::RedisSet.new options
     end
   end
-end
+end

data/lib/polipus/url_tracker/bloomfilter.rb

@@ -1,4 +1,4 @@
-require "redis-bloomfilter"
+require 'redis-bloomfilter'
 module Polipus
   module UrlTracker
     class Bloomfilter
@@ -10,18 +10,17 @@ module Polipus
         @bf.include?(url)
       end
 
-      def visit url
+      def visit(url)
         @bf.insert url
       end
 
-      def remove url
+      def remove(url)
         @bf.remove url
       end
 
       def clear
         @bf.clear
       end
-
     end
   end
-end
+end

data/lib/polipus/url_tracker/redis_set.rb

@@ -1,21 +1,20 @@
 module Polipus
   module UrlTracker
     class RedisSet
-      
       def initialize(options = {})
         @redis    = options[:redis] || Redis.current
         @set_name = options[:key_name]
       end
 
       def visited?(url)
-        @redis.sismember(@set_name,url)
+        @redis.sismember(@set_name, url)
       end
 
-      def visit url
+      def visit(url)
         @redis.sadd(@set_name, url)
       end
 
-      def remove url
+      def remove(url)
         @redis.srem(@set_name, url, 0)
       end