polipus 0.4.0 → 0.5.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NGU0MWZlMWIwMGM2MWNhMmJiOTU3MDM0YWI0ZjY0MzI3NmNkYzgyNA==
-  data.tar.gz: !binary |-
-    MTAxYjZmYWNjMTlkYzk3Y2ZhNjdjMDFmODM2OTQ5YmQ3ZDcyNTQzNw==
+SHA1:
+  metadata.gz: 0e3c15f4ab5c3b4eca8f4703b80d7d8fa74210c3
+  data.tar.gz: ae0dd32d81bbcbef350949e0c51fecaaf59ac8ff
 SHA512:
-  metadata.gz: !binary |-
-    ZDE5NDExNmE1M2M4MDM0MTExNzVkNmM0ZmVjNTQyOTg5Y2JkYTdmZjBhY2Fi
-    NmM1MDc5ZTI0ODkxNzlhY2ZkZGVjZjI2ZTMyNjZhNTMwOWZhZjQ2OGY4ZTJl
-    ZmI4ZjUwNmQ3YjAxOTdlODI2MWRmMWUzNjY3Yzk1MmNiZTk0Y2E=
-  data.tar.gz: !binary |-
-    YjNlYTZlMjU5OTBmN2ZjMTI1OTY1MzgzNzBhYjBjYTAzN2MwYzY0M2U1ZGYw
-    MjcyMmMxM2I1ZTY5OWEzZDQ3MjQ1OTgzMzcyN2I5NGQ5NTAzZTEzY2FmMDRl
-    MmJiZDdjOTFmMjQwZWU1MzM1OGJhN2E4NTRhNGExZTAyMTVmN2I=
+  metadata.gz: c3a57d1cbfbab4c77cf3b746c7b73eb84cf675e7998eb5979af6d17d997ee26781803e519eae50de6762590fc7287d09ccf4ac0d640dff40602b8a507a356de2
+  data.tar.gz: 743b128ee0dd8fefbbdd1bd7443afbc4a2c62dd2f3d3751d3d693719e9a8e2e145668ccd58945b77f5f780f50757a11248df2b1963c8e7cc35706e937627e5e0

data/.rubocop_todo.yml

@@ -1,25 +1,36 @@
 # This configuration was generated by `rubocop --auto-gen-config`
-# on 2014-06-08 11:25:39 -0700 using RuboCop version 0.23.0.
+# on 2015-07-08 20:22:49 -0700 using RuboCop version 0.29.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
 # Offense count: 1
-Style/ClassVars:
+Lint/HandleExceptions:
   Enabled: false
 
-# Offense count: 10
+# Offense count: 21
+Metrics/AbcSize:
+  Max: 103
+
+# Offense count: 12
 Metrics/CyclomaticComplexity:
-  Max: 16
+  Max: 15
 
-# Offense count: 26
-Style/Documentation:
+# Offense count: 10
+Metrics/PerceivedComplexity:
+  Max: 17
+
+# Offense count: 1
+Style/ClassVars:
   Enabled: false
 
+# Offense count: 27
+Style/Documentation:
+  Enabled: false
 
 # Offense count: 2
-# Configuration parameters: EnforcedStyle, SupportedStyles.
+# Configuration parameters: EnforcedStyle, MinBodyLength, SupportedStyles.
 Style/Next:
   Enabled: false
 
@@ -28,6 +39,10 @@ Style/Next:
 Style/RegexpLiteral:
   Enabled: false
 
-# Offense count: 4
+# Offense count: 3
 Style/RescueModifier:
   Enabled: false
+
+# Offense count: 1
+Style/UnlessElse:
+  Enabled: false

data/AUTHORS.md

@@ -3,3 +3,4 @@
 * [Francesco Laurita](francesco.laurita@gmail.com)
 * [Tobias L. Maier](http://tobiasmaier.info/)
 * [Marcos Piccinini](https://github.com/nofxx)
+* [Martin Bianculli](https://github.com/lepek)

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.5.0 (2015-07-08)
+
+[Compare changes in gem](https://github.com/taganaka/polipus/compare/0.4.0...0.5.0)
+
+* Support for MongoDB driver ~> 2.0.6 has been added
+* Minor code cleanup
+
 ## 0.4.0 (2015-01-12)
 
 [Compare changes in gem](https://github.com/taganaka/polipus/compare/0.3.3...0.4.0)

data/lib/polipus.rb

@@ -129,7 +129,14 @@ module Polipus
 
       @urls = [urls].flatten.map { |url| URI(url) }
       @urls.each { |url| url.path = '/' if url.path.empty? }
-      @robots = Polipus::Robotex.new(@options[:user_agent]) if @options[:obey_robots_txt]
+      if @options[:obey_robots_txt]
+        @robots =
+          if @options[:user_agent].respond_to?(:sample)
+            Polipus::Robotex.new(@options[:user_agent].sample)
+          else
+            Polipus::Robotex.new(@options[:user_agent])
+          end
+      end
       # Attach signal handling if enabled
       SignalHandler.enable if @options[:enable_signal_handler]
 
@@ -170,7 +177,6 @@ module Polipus
           http  =  HTTP.new(@options)
           queue =  queue_factory
           queue.process(false, @options[:queue_timeout]) do |message|
-
             next if message.nil?
 
             execute_plugin 'on_message_received'
@@ -199,7 +205,7 @@ module Polipus
               rurls = pages.map { |e| e.url.to_s }.join(' --> ')
               @logger.info { "Got redirects! #{rurls}" }
               page = pages.pop
-              page.aliases = pages.map { |e| e.url }
+              page.aliases = pages.map(&:url)
               if page_exists? page
                 @logger.info { "[worker ##{worker_number}] Page (#{page.url}) already stored." }
                 queue.commit
@@ -253,7 +259,7 @@ module Polipus
         end
       end
 
-      @workers_pool.each { |w| w.join }
+      @workers_pool.each(&:join)
       @on_crawl_end.each { |e| e.call(self) }
       execute_plugin 'on_crawl_end'
     end
@@ -474,7 +480,7 @@ module Polipus
         next unless p.respond_to?(method)
         @logger.info { "Running plugin method #{method} on #{k}" }
         ret_val = p.send(method, self)
-        instance_eval(&ret_val) if ret_val.kind_of? Proc
+        instance_eval(&ret_val) if ret_val.is_a? Proc
       end
     end
   end

data/lib/polipus/http.rb

@@ -80,7 +80,11 @@ module Polipus
     # or nil if no such option is set
     #
     def user_agent
-      @opts[:user_agent]
+      if @opts[:user_agent].respond_to?(:sample)
+        @opts[:user_agent].sample
+      else
+        @opts[:user_agent]
+      end
     end
 
     #
@@ -108,7 +112,7 @@ module Polipus
     # The proxy password
     #
     def proxy_pass
-      #return proxy_host_port[3] unless @opts[:proxy_host_port].nil?
+      # return proxy_host_port[3] unless @opts[:proxy_host_port].nil?
       @opts[:proxy_pass].respond_to?(:call) ? @opts[:proxy_pass].call(self) : @opts[:proxy_pass]
     end
 
@@ -162,7 +166,13 @@ module Polipus
 
         response, response_time = get_response(loc, referer)
         code = Integer(response.code)
-        redirect_to = response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
+        redirect_to =
+          begin
+            response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
+          rescue URI::InvalidURIError => e
+            @opts[:logger].debug { "Request #{url} got #{e}" } if @opts[:logger]
+            nil
+          end
         yield response, code, loc, redirect_to, response_time
         limit -= 1
         break unless (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
@@ -233,7 +243,7 @@ module Polipus
 
       # Block has higher priority
       unless @opts[:proxy_host_port].nil?
-        p_host, p_port, p_user, p_pass = proxy_host_port 
+        p_host, p_port, p_user, p_pass = proxy_host_port
       else
         p_host = proxy_host
         p_port = proxy_port

data/lib/polipus/page.rb

@@ -3,6 +3,8 @@ require 'nokogiri'
 require 'json'
 require 'ostruct'
 require 'set'
+require 'kconv'
+
 module Polipus
   class Page
     # The URL of the page
@@ -70,7 +72,7 @@ module Polipus
         u = a['href']
         next if u.nil? || u.empty?
         abs = to_absolute(u) rescue next
-        @links << abs if in_domain?(abs)
+        @links << abs if abs && in_domain?(abs)
       end
       @links.to_a
     end
@@ -80,7 +82,10 @@ module Polipus
     #
     def doc
       return @doc if @doc
-      @doc = Nokogiri::HTML(@body) if @body && html? rescue nil
+      @body ||= ''
+      @body = @body.encode('utf-8', 'binary', invalid: :replace,
+                                              undef: :replace, replace: '')
+      @doc = Nokogiri::HTML(@body.toutf8, nil, 'utf-8') if @body && html?
     end
 
     #
@@ -166,10 +171,22 @@ module Polipus
     def to_absolute(link)
       return nil if link.nil?
 
+      valid_link = link.to_s.encode('utf-8', 'binary', invalid: :replace,
+                                                       undef: :replace, replace: '')
+
       # remove anchor
-      link = URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/, '')))
+      link =
+        begin
+          URI.encode(URI.decode(valid_link.gsub(/#[a-zA-Z0-9_-]*$/, '')))
+        rescue URI::Error
+          return nil
+        end
 
-      relative = URI(link)
+      relative = begin
+                   URI(link)
+                 rescue URI::Error
+                   return nil
+                 end
       absolute = base ? base.merge(relative) : @url.merge(relative)
 
       absolute.path = '/' if absolute.path.empty?

data/lib/polipus/queue_overflow.rb

@@ -5,15 +5,15 @@ module Polipus
   module QueueOverflow
     def self.mongo_queue(mongo_db, queue_name, options = {})
       require 'polipus/queue_overflow/mongo_queue'
-      mongo_db ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
-      fail 'First argument must be an instance of Mongo::DB' unless mongo_db.is_a?(Mongo::DB)
+      mongo_db ||= Mongo::Client.new(['localhost:27_017'], database: 'polipus')
+      fail 'First argument must be an instance of Mongo::Client' unless mongo_db.is_a?(Mongo::Client)
       self::MongoQueue.new mongo_db, queue_name, options
     end
 
     def self.mongo_queue_capped(mongo_db, queue_name, options = {})
       require 'polipus/queue_overflow/mongo_queue_capped'
-      mongo_db ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
-      fail 'First argument must be an instance of Mongo::DB' unless mongo_db.is_a?(Mongo::DB)
+      mongo_db ||= Mongo::Client.new(['localhost:27_017'], database: 'polipus')
+      fail 'First argument must be an instance of Mongo::Client' unless mongo_db.is_a?(Mongo::Client)
       options[:max] = 1_000_000 if options[:max].nil?
       self::MongoQueueCapped.new mongo_db, queue_name, options
     end

data/lib/polipus/queue_overflow/mongo_queue.rb

@@ -14,7 +14,7 @@ module Polipus
       end
 
       def length
-        @mongo_db[@collection_name].count
+        @mongo_db[@collection_name].find.count
       end
 
       def empty?
@@ -28,18 +28,18 @@ module Polipus
 
       def push(data)
         if @options[:ensure_uniq]
-          @mongo_db[@collection_name].update({ payload: data }, { payload: data }, { upsert: true, w: 1 })
+          @mongo_db[@collection_name].find(payload: data).replace_one({ payload: data }, upsert: true)
         else
-          @mongo_db[@collection_name].insert(payload: data)
+          @mongo_db[@collection_name].insert_one(payload: data)
         end
         true
       end
 
       def pop(_ = false)
         @semaphore.synchronize do
-          doc = @mongo_db[@collection_name].find({}, sort: { _id: 1 }).limit(1).first
+          doc = @mongo_db[@collection_name].find.sort(_id: 1).limit(1).first
           return nil if doc.nil?
-          @mongo_db[@collection_name].remove(_id: doc['_id'])
+          @mongo_db[@collection_name].find(_id: doc['_id']).delete_one
           doc && doc['payload'] ? doc['payload'] : nil
         end
       end
@@ -53,7 +53,8 @@ module Polipus
       protected
 
       def ensure_index
-        @mongo_db[@collection_name].ensure_index({ payload: 1 }, { background: 1, unique: 1, drop_dups: 1 })
+        # @TODO: Drop dups option was removed. We may want to add something here to remove duplications
+        @mongo_db[@collection_name].indexes.create_one({ payload: 1 }, background: true, unique: true)
       end
     end
   end

data/lib/polipus/queue_overflow/mongo_queue_capped.rb

@@ -13,8 +13,8 @@ module Polipus
         @semaphore.synchronize do
           s = size
           if s > @max
-            docs = @mongo_db[@collection_name].find({}, { sort: { _id: 1 }, fields: [:_id] }).limit(s - @max).map { |e| e['_id'] }
-            @mongo_db[@collection_name].remove(:_id => { '$in' => docs }, '$isolated' => 1)
+            docs = @mongo_db[@collection_name].find.sort(_id: 1).projection(_id: 1).limit(s - @max).map { |e| e['_id'] }
+            @mongo_db[@collection_name].find(_id: { '$in' => docs }).delete_many
           end
         end
       end

data/lib/polipus/storage.rb

@@ -6,14 +6,14 @@ module Polipus
 
     def self.mongo_store(mongo = nil, collection = COLLECTION, except = [])
       require 'polipus/storage/mongo_store'
-      mongo ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
-      fail 'First argument must be an instance of Mongo::DB' unless mongo.is_a?(Mongo::DB)
+      mongo ||= Mongo::Client.new(['localhost:27_017'], database: 'polipus')
+      fail 'First argument must be an instance of Mongo::Client' unless mongo.is_a?(Mongo::Client)
       self::MongoStore.new(mongo: mongo, collection: collection, except: except)
     end
 
     def self.rethink_store(conn = nil, table = COLLECTION, except = [])
       require 'polipus/storage/rethink_store'
-      conn ||= RethinkDB::RQL.new.connect(host: 'localhost', port: 28_015, db: 'polipus' )
+      conn ||= RethinkDB::RQL.new.connect(host: 'localhost', port: 28_015, db: 'polipus')
       fail "First argument must be a RethinkDB::Connection, got `#{conn.class}`" unless conn.is_a?(RethinkDB::Connection)
       self::RethinkStore.new(conn: conn, table: table,  except: except)
     end

data/lib/polipus/storage/memory_store.rb

@@ -48,7 +48,7 @@ module Polipus
 
       def clear
         @semaphore.synchronize do
-          @store = Hash.new
+          @store = {}
         end
       end
     end

data/lib/polipus/storage/mongo_store.rb

@@ -2,6 +2,7 @@
 require 'mongo'
 require 'zlib'
 require 'thread'
+require 'pry'
 
 module Polipus
   module Storage
@@ -10,10 +11,9 @@ module Polipus
       def initialize(options = {})
         @mongo      = options[:mongo]
         @collection = options[:collection]
-        @mongo.create_collection(@collection)
         begin
           @mongo[@collection].ensure_index(:uuid, unique: true, dropDups: true, background: true)
-        rescue Exception
+        rescue StandardError
         end
 
         @compress_body = options[:compress_body] ||= true
@@ -28,16 +28,20 @@ module Polipus
           obj['uuid'] = uuid(page)
           obj['body'] = Zlib::Deflate.deflate(obj['body']) if @compress_body && obj['body']
           BINARY_FIELDS.each do |field|
-            obj[field] = BSON::Binary.new(obj[field]) unless obj[field].nil?
+            obj[field] = BSON::Binary.new(obj[field].force_encoding('UTF-8').encode('UTF-8')) unless obj[field].nil?
           end
-          @mongo[@collection].update({ uuid: obj['uuid'] }, obj, upsert: true, w: 1)
+
+          # We really need 2.0.6+ version for this to work
+          # https://jira.mongodb.org/browse/RUBY-881
+          @mongo[@collection].find(uuid: uuid(page)).replace_one(obj, upsert: true)
+
           obj['uuid']
         end
       end
 
       def exists?(page)
         @semaphore.synchronize do
-          doc = @mongo[@collection].find({ uuid: uuid(page) }, { fields: [:_id] }).limit(1).first
+          doc = @mongo[@collection].find(uuid: uuid(page)).projection(_id: 1).limit(1).first
           !doc.nil?
         end
       end
@@ -51,16 +55,16 @@ module Polipus
 
       def remove(page)
         @semaphore.synchronize do
-          @mongo[@collection].remove(uuid: uuid(page))
+          @mongo[@collection].find(uuid: uuid(page)).delete_one
         end
       end
 
       def count
-        @mongo[@collection].count
+        @mongo[@collection].find.count
       end
 
       def each
-        @mongo[@collection].find({}, timeout: false) do |cursor|
+        @mongo[@collection].find.no_cursor_timeout do |cursor|
           cursor.each do |doc|
             page = load_page(doc)
             yield doc['uuid'], page
@@ -76,7 +80,7 @@ module Polipus
 
       def load_page(hash)
         BINARY_FIELDS.each do |field|
-          hash[field] = hash[field].to_s
+          hash[field] = hash[field].data unless hash[field].nil?
         end
         hash['body'] = Zlib::Inflate.inflate(hash['body']) if @compress_body && hash['body'] && !hash['body'].empty?
         page = Page.from_hash(hash)

data/lib/polipus/storage/rethink_store.rb

@@ -12,6 +12,9 @@ module Polipus
         @rethink = options[:conn]
         @table   = options[:table]
 
+        unless @r.db_list.run(@rethink).include?(@rethink.default_db)
+          @r.db_create(@rethink.default_db).run(@rethink)
+        end
         unless @r.table_list.run(@rethink).include?(@table)
           @r.table_create(@table).run(@rethink)
           @r.table(@table).index_create('created_at')

data/lib/polipus/version.rb

@@ -1,5 +1,5 @@
 # encoding: UTF-8
 module Polipus
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
   HOMEPAGE = 'https://github.com/taganaka/polipus'
 end

data/polipus.gemspec

@@ -8,10 +8,10 @@ Gem::Specification.new do |s|
   s.authors     = ['Francesco Laurita']
   s.email       = ['francesco.laurita@gmail.com']
   s.homepage    = Polipus::HOMEPAGE
-  s.summary     = %q(Polipus distributed web-crawler framework)
-  s.description = %q(
+  s.summary     = 'Polipus distributed web-crawler framework'
+  s.description = '
     An easy to use distributed web-crawler framework based on Redis
-  )
+  '
   s.licenses    = ['MIT']
   s.platform    = Gem::Platform::RUBY
 
@@ -30,8 +30,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'redis-queue', '~> 0.0', '>= 0.0.4'
   s.add_runtime_dependency 'redis-bloomfilter', '~> 0.0', '>= 0.0.3'
 
-  s.add_development_dependency 'mongo', '~>1.11.0'
-  s.add_development_dependency 'rethinkdb', '~>1.15.0'
+  s.add_development_dependency 'mongo', '~> 2.0.6'
+  s.add_development_dependency 'rethinkdb', '~> 1.15.0'
 
   s.add_development_dependency 'rake', '~> 10.3'
   s.add_development_dependency 'rspec', '~> 3.1.0'
@@ -41,4 +41,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'webmock', '~> 1.20.0'
 
   s.add_development_dependency 'coveralls'
+
+  s.add_development_dependency 'pry'
 end

data/spec/polipus/http_spec.rb

@@ -108,4 +108,32 @@ describe Polipus::HTTP do
       end
     end
   end
+
+  describe 'random user_agent' do
+    context 'when user_agent is string' do
+      it '#user_agent' do
+        http = Polipus::HTTP.new(open_timeout: 1, read_timeout: 1, user_agent: 'Googlebot')
+        expect(http.user_agent).to eq('Googlebot')
+      end
+    end
+
+    context 'when user_agent is list' do
+      let(:user_agents) do
+        ['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1',
+         'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2',
+         'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7',
+         'Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre',
+         'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10',
+         'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)',
+         'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 GTB5',
+         'Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)'
+        ]
+      end
+
+      it '#user_agent' do
+        http = Polipus::HTTP.new(open_timeout: 1, read_timeout: 1, user_agent: user_agents)
+        expect(user_agents).to include(http.user_agent)
+      end
+    end
+  end
 end

data/spec/polipus/queue_overflow/manager_spec.rb

@@ -6,7 +6,7 @@ require 'redis-queue'
 
 describe Polipus::QueueOverflow::Manager do
   before(:all) do
-    @mongo = Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('_test_polipus')
+    @mongo = Mongo::Client.new(['localhost:27_017'], database: '_test_polipus')
     @mongo['_test_pages'].drop
     @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
     @redis_q = Redis::Queue.new('queue_test', 'bp_queue_test', redis: Redis.new)

data/spec/polipus/storage/memory_store_spec.rb

@@ -69,7 +69,7 @@ describe Polipus::Storage::MemoryStore do
     p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
     storage.add p
     p = storage.get p
-    expect(p.body).to be_empty
+    expect(p.body).to be_nil
     storage.clear
   end
 

data/spec/polipus/storage/mongo_store_spec.rb

@@ -4,7 +4,7 @@ require 'mongo'
 require 'polipus/storage/mongo_store'
 describe Polipus::Storage::MongoStore do
   before(:all)do
-    @mongo = Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('_test_polipus')
+    @mongo = Mongo::Client.new(['localhost:27_017'], database: '_test_polipus')
     @mongo['_test_pages'].drop
     @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
   end
@@ -21,8 +21,8 @@ describe Polipus::Storage::MongoStore do
     p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
     uuid = @storage.add p
     expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
-    expect(@storage.count).to be 1
-    expect(@mongo['_test_pages'].count).to be 1
+    expect(@storage.count.to_i).to be 1
+    expect(@mongo['_test_pages'].find.count.to_i).to be 1
     p = @storage.get p
     expect(p.url.to_s).to eq('http://www.google.com')
     expect(p.body).to eq('<html></html>')
@@ -33,7 +33,7 @@ describe Polipus::Storage::MongoStore do
     @storage.add p
     p = @storage.get p
     expect(p.code).to eq(301)
-    expect(@mongo['_test_pages'].count).to be 1
+    expect(@mongo['_test_pages'].find.count.to_i).to be 1
   end
 
   it 'should iterate over stored pages' do
@@ -47,7 +47,7 @@ describe Polipus::Storage::MongoStore do
     p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
     @storage.remove p
     expect(@storage.get(p)).to be_nil
-    expect(@storage.count).to be 0
+    expect(@storage.count.to_i).to be 0
   end
 
   it 'should store a page removing a query string from the uuid generation' do
@@ -83,7 +83,7 @@ describe Polipus::Storage::MongoStore do
     p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
     storage.add p
     p = storage.get p
-    expect(p.body).to be_empty
+    expect(p.body).to be_nil
     storage.clear
   end
 

data/spec/polipus_spec.rb

@@ -93,5 +93,14 @@ describe Polipus::PolipusCrawler do
       polipus.takeover
       polipus.storage.each { |_id, page| expect(page.url.path =~ /$\/downloads\//).to be_falsey }
     end
+
+    it 'should obey to the robots.txt file with list user_agent' do
+      user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)'
+      lopt = p_options
+      lopt[:obey_robots_txt] = true
+      lopt[:user_agent] = [user_agent]
+      flexmock(Polipus::Robotex).should_receive(:new).with(user_agent)
+      Polipus::PolipusCrawler.new('polipus-rspec', ['https://rubygems.org/gems/polipus'], lopt)
+    end
   end
 end

metadata

@@ -1,248 +1,262 @@
 --- !ruby/object:Gem::Specification
 name: polipus
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Francesco Laurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-26 00:00:00.000000000 Z
+date: 2015-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: http-cookie
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.4
 - !ruby/object:Gem::Dependency
   name: hiredis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.5
 - !ruby/object:Gem::Dependency
   name: redis-queue
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.4
 - !ruby/object:Gem::Dependency
   name: redis-bloomfilter
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.3
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.11.0
+        version: 2.0.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.11.0
+        version: 2.0.6
 - !ruby/object:Gem::Dependency
   name: rethinkdb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.15.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.15.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: flexmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.9.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.9.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.20.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.20.0
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ! "\n    An easy to use distributed web-crawler framework based on Redis\n
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: "\n    An easy to use distributed web-crawler framework based on Redis\n
   \ "
 email:
 - francesco.laurita@gmail.com
@@ -250,12 +264,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .document
-- .gitignore
-- .rspec
-- .rubocop.yml
-- .rubocop_todo.yml
-- .travis.yml
+- ".document"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".travis.yml"
 - AUTHORS.md
 - CHANGELOG.md
 - Gemfile
@@ -330,17 +344,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: polipus
-rubygems_version: 2.4.5
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Polipus distributed web-crawler framework
@@ -371,4 +385,3 @@ test_files:
 - spec/polipus/url_tracker_spec.rb
 - spec/polipus_spec.rb
 - spec/spec_helper.rb
-has_rdoc: