polipus 0.3.0 → 0.3.1

data/spec/queue_overflow_spec.rb

@@ -1,12 +1,12 @@
-require "spec_helper"
-require "polipus/queue_overflow"
+require 'spec_helper'
+require 'polipus/queue_overflow'
 
 describe Polipus::QueueOverflow do
 
   before(:all) do
-    @queue_overflow        = Polipus::QueueOverflow.mongo_queue(nil, "queue_test")
-    @queue_overflow_capped = Polipus::QueueOverflow.mongo_queue_capped(nil, "queue_test_c", {:max => 20})
-    @queue_overflow_uniq   = Polipus::QueueOverflow.mongo_queue(nil, "queue_test_u", {:ensure_uniq => true })
+    @queue_overflow        = Polipus::QueueOverflow.mongo_queue(nil, 'queue_test')
+    @queue_overflow_capped = Polipus::QueueOverflow.mongo_queue_capped(nil, 'queue_test_c', max: 20)
+    @queue_overflow_uniq   = Polipus::QueueOverflow.mongo_queue(nil, 'queue_test_u', ensure_uniq: true)
 
   end
 
@@ -26,45 +26,45 @@ describe Polipus::QueueOverflow do
     [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
       q.empty?.should be_true
       q.pop.should be_nil
-      q << "test"
+      q << 'test'
       q.size.should be == 1
-      q.pop.should be == "test"
+      q.pop.should be == 'test'
       q.empty?.should be_true
       q.pop.should be_nil
       q.size.should be == 0
       q.empty?.should be_true
     end
-    
+
   end
 
   it 'should act as a queue' do
     [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
       10.times { |i| q << "message_#{i}" }
       q.size.should be == 10
-      q.pop.should be == "message_0"
+      q.pop.should be == 'message_0'
     end
-  
+
   end
 
   it 'should work with complex paylod' do
     [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
-      a = {'a' => [1,2,3], 'b' => 'a_string'}
+      a = { 'a' => [1, 2, 3], 'b' => 'a_string' }
       q << a.to_json
       b = q.pop
       JSON.parse(b).should be == a
     end
-    
+
   end
 
   it 'should honor max items if it is capped' do
     30.times { |i| @queue_overflow_capped << "message_#{i}" }
     @queue_overflow_capped.size.should be == 20
-    @queue_overflow_capped.pop.should be == "message_10"
+    @queue_overflow_capped.pop.should be == 'message_10'
   end
 
   it 'should contains only unique items' do
-    20.times {@queue_overflow_uniq << "A"}
-    20.times {@queue_overflow_uniq << "B"}
+    20.times { @queue_overflow_uniq << 'A' }
+    20.times { @queue_overflow_uniq << 'B' }
     @queue_overflow_uniq.size.should be == 2
   end
 

data/spec/robotex_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
-require "polipus/robotex"
+require 'polipus/robotex'
 describe Polipus::Robotex do
-  let(:spec_domain){"http://www.example.com/"}
+  let(:spec_domain) { 'http://www.example.com/' }
   before(:each) do
     robots = <<-END
 User-Agent: msnbot
@@ -18,9 +18,8 @@ Disallow: /locked
 Allow: /locked
 END
     stub_request(:get, 'http://www.example.com/robots.txt')
-    .to_return(:body => robots, :status => [200, "OK"], :headers => { "Content-Type" => 'text/plain' })
+    .to_return(body: robots, status: [200, 'OK'], headers: { 'Content-Type' => 'text/plain' })
   end
-  
 
   describe '#initialize' do
     context 'when no arguments are supplied' do
@@ -71,16 +70,16 @@ END
     context 'when no Crawl-Delay is specified for the user-agent' do
       it 'returns nil' do
         robotex = Polipus::Robotex.new
-        robotex.delay(spec_domain).should be_nil 
+        robotex.delay(spec_domain).should be_nil
       end
 
-    context 'when Crawl-Delay is specified for the user-agent' do
-      it 'returns the delay as a Fixnum' do
-        robotex = Polipus::Robotex.new('msnbot')
-        robotex.delay(spec_domain).should == 20
+      context 'when Crawl-Delay is specified for the user-agent' do
+        it 'returns the delay as a Fixnum' do
+          robotex = Polipus::Robotex.new('msnbot')
+          robotex.delay(spec_domain).should == 20
+        end
       end
     end
-    end
   end
 
 end

data/spec/signal_handler_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe Polipus::SignalHandler do
+
+  context 'signal handler' do
+
+    it 'should be enabled by default' do
+      Polipus::PolipusCrawler.new('polipus-rspec', [])
+      Polipus::SignalHandler.enabled?.should be true
+    end
+
+    it 'should be disabled if specified' do
+      Polipus::PolipusCrawler.new('polipus-rspec', [], enable_signal_handler: false)
+      Polipus::SignalHandler.enabled?.should be false
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -4,9 +4,9 @@
 # loaded once.
 #
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
-require "digest/md5"
-require "coveralls"
-require "webmock/rspec"
+require 'digest/md5'
+require 'coveralls'
+require 'webmock/rspec'
 
 Coveralls.wear!
 
@@ -26,14 +26,15 @@ RSpec.configure do |config|
       example.run
     end
   end
+  config.before(:each) { Polipus::SignalHandler.disable }
 end
-require "vcr"
-require "polipus"
+require 'vcr'
+require 'polipus'
 VCR.configure do |c|
   c.cassette_library_dir = "#{File.dirname(__FILE__)}/cassettes"
   c.hook_into :webmock
 end
 
-def page_factory url, params = {}
+def page_factory(url, params = {})
   Polipus::Page.new url, params
 end

data/spec/storage_memory_spec.rb

@@ -1,12 +1,12 @@
-require "spec_helper"
-require "mongo"
-require "polipus/storage/memory_store"
+require 'spec_helper'
+require 'mongo'
+require 'polipus/storage/memory_store'
 describe Polipus::Storage::MemoryStore do
-  
-  let(:storage){Polipus::Storage.memory_store}
+
+  let(:storage) { Polipus::Storage.memory_store }
 
   it 'should store a page' do
-    p = page_factory 'http://www.google.com', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
     uuid = storage.add p
     uuid.should be == 'ed646a3334ca891fd3467db131372140'
     storage.count.should be 1
@@ -16,7 +16,7 @@ describe Polipus::Storage::MemoryStore do
   end
 
   it 'should update a page' do
-    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
     storage.add p
     p = storage.get p
     p.code.should be == 301
@@ -24,21 +24,21 @@ describe Polipus::Storage::MemoryStore do
 
   it 'should iterate over stored pages' do
     storage.each do |k, page|
-      k.should be == "ed646a3334ca891fd3467db131372140"
+      k.should be == 'ed646a3334ca891fd3467db131372140'
       page.url.to_s.should be == 'http://www.google.com'
     end
   end
 
   it 'should delete a page' do
-    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
     storage.remove p
     storage.get(p).should be_nil
     storage.count.should be 0
   end
 
   it 'should store a page removing a query string from the uuid generation' do
-    p = page_factory 'http://www.asd.com/?asd=lol', :code => 200, :body => '<html></html>'
-    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
     storage.include_query_string_in_uuid = false
     storage.add p
     storage.exists?(p_no_query).should be_true
@@ -46,8 +46,8 @@ describe Polipus::Storage::MemoryStore do
   end
 
   it 'should store a page removing a query string from the uuid generation no ending slash' do
-    p = page_factory 'http://www.asd.com?asd=lol', :code => 200, :body => '<html></html>'
-    p_no_query = page_factory 'http://www.asd.com', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
     storage.include_query_string_in_uuid = false
     storage.add p
     storage.exists?(p_no_query).should be_true
@@ -55,7 +55,7 @@ describe Polipus::Storage::MemoryStore do
   end
 
   it 'should store a page with user data associated' do
-    p = page_factory 'http://www.user.com',  :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
     p.user_data.name = 'Test User Data'
     storage.add p
     storage.exists?(p).should be_true
@@ -66,7 +66,7 @@ describe Polipus::Storage::MemoryStore do
 
   it 'should honor the except parameters' do
     storage = Polipus::Storage.mongo_store(@mongo, '_test_pages', ['body'])
-    p = page_factory 'http://www.user-doo.com',  :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
     storage.add p
     p = storage.get p
     p.body.should be_empty
@@ -75,15 +75,15 @@ describe Polipus::Storage::MemoryStore do
 
   it 'should return false if a doc not exists' do
     storage.include_query_string_in_uuid = false
-    p_other  = page_factory 'http://www.asdrrrr.com', :code => 200, :body => '<html></html>'
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
     storage.exists?(p_other).should be_false
     storage.add p_other
     storage.exists?(p_other).should be_true
-    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', :code => 200, :body => '<html></html>'
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
     storage.exists?(p_other).should be_true
     storage.include_query_string_in_uuid = true
     storage.exists?(p_other).should be_false
 
   end
 
-end
+end

data/spec/storage_mongo_spec.rb

@@ -1,9 +1,9 @@
-require "spec_helper"
-require "mongo"
-require "polipus/storage/mongo_store"
+require 'spec_helper'
+require 'mongo'
+require 'polipus/storage/mongo_store'
 describe Polipus::Storage::MongoStore do
   before(:all)do
-    @mongo = Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('_test_polipus')
+    @mongo = Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('_test_polipus')
     @mongo['_test_pages'].drop
     @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
   end
@@ -17,7 +17,7 @@ describe Polipus::Storage::MongoStore do
   end
 
   it 'should store a page' do
-    p = page_factory 'http://www.google.com', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
     uuid = @storage.add p
     uuid.should be == 'ed646a3334ca891fd3467db131372140'
     @storage.count.should be 1
@@ -28,7 +28,7 @@ describe Polipus::Storage::MongoStore do
   end
 
   it 'should update a page' do
-    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
     @storage.add p
     p = @storage.get p
     p.code.should be == 301
@@ -37,21 +37,21 @@ describe Polipus::Storage::MongoStore do
 
   it 'should iterate over stored pages' do
     @storage.each do |k, page|
-      k.should be == "ed646a3334ca891fd3467db131372140"
+      k.should be == 'ed646a3334ca891fd3467db131372140'
       page.url.to_s.should be == 'http://www.google.com'
     end
   end
 
   it 'should delete a page' do
-    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
     @storage.remove p
     @storage.get(p).should be_nil
     @storage.count.should be 0
   end
 
   it 'should store a page removing a query string from the uuid generation' do
-    p = page_factory 'http://www.asd.com/?asd=lol', :code => 200, :body => '<html></html>'
-    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
     @storage.include_query_string_in_uuid = false
     @storage.add p
     @storage.exists?(p_no_query).should be_true
@@ -59,8 +59,8 @@ describe Polipus::Storage::MongoStore do
   end
 
   it 'should store a page removing a query string from the uuid generation no ending slash' do
-    p = page_factory 'http://www.asd.com?asd=lol', :code => 200, :body => '<html></html>'
-    p_no_query = page_factory 'http://www.asd.com', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
     @storage.include_query_string_in_uuid = false
     @storage.add p
     @storage.exists?(p_no_query).should be_true
@@ -68,7 +68,7 @@ describe Polipus::Storage::MongoStore do
   end
 
   it 'should store a page with user data associated' do
-    p = page_factory 'http://www.user.com',  :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
     p.user_data.name = 'Test User Data'
     @storage.add p
     @storage.exists?(p).should be_true
@@ -79,7 +79,7 @@ describe Polipus::Storage::MongoStore do
 
   it 'should honor the except parameters' do
     storage = Polipus::Storage.mongo_store(@mongo, '_test_pages', ['body'])
-    p = page_factory 'http://www.user-doo.com',  :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
     storage.add p
     p = storage.get p
     p.body.should be_empty
@@ -88,11 +88,11 @@ describe Polipus::Storage::MongoStore do
 
   it 'should return false if a doc not exists' do
     @storage.include_query_string_in_uuid = false
-    p_other  = page_factory 'http://www.asdrrrr.com', :code => 200, :body => '<html></html>'
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
     @storage.exists?(p_other).should be_false
     @storage.add p_other
     @storage.exists?(p_other).should be_true
-    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', :code => 200, :body => '<html></html>'
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
     @storage.exists?(p_other).should be_true
     @storage.include_query_string_in_uuid = true
     @storage.exists?(p_other).should be_false
@@ -101,7 +101,7 @@ describe Polipus::Storage::MongoStore do
 
   it 'should set page.fetched_at based on the id creation' do
     storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
-    p = page_factory 'http://www.user-doojo.com',  :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.user-doojo.com',  code: 200, body: '<html></html>'
     storage.add p
     p.fetched_at.should be_nil
     p = storage.get p
@@ -110,11 +110,11 @@ describe Polipus::Storage::MongoStore do
 
   it 'should NOT set page.fetched_at if already present' do
     storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
-    p = page_factory 'http://www.user-doojooo.com',  :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.user-doojooo.com',  code: 200, body: '<html></html>'
     p.fetched_at = 10
     storage.add p
     p = storage.get p
     p.fetched_at.should be 10
   end
 
-end
+end

data/spec/storage_s3_spec.rb

@@ -1,24 +1,23 @@
-require "spec_helper"
-require "aws/s3"
-require "polipus/storage/s3_store"
+require 'spec_helper'
+require 'aws/s3'
+require 'polipus/storage/s3_store'
 describe Polipus::Storage::S3Store do
-  
+
   before(:each) do
     @storage = Polipus::Storage.s3_store(
-      '_test_pages', 
-      {
-        :access_key_id => 'XXXXXXX', 
-        :secret_access_key => 'XXXX'
-      }
+      '_test_pages',
+
+      access_key_id: 'XXXXXXX',
+      secret_access_key: 'XXXX'
+
     )
   end
-  
-  after(:each) {@storage.clear}
-  
+
+  after(:each) { @storage.clear }
 
   it 'should store a page' do
 
-    p = page_factory 'http://www.google.com', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
     uuid = @storage.add p
     uuid.should be == 'ed646a3334ca891fd3467db131372140'
     @storage.count.should be 1
@@ -30,7 +29,7 @@ describe Polipus::Storage::S3Store do
   end
 
   it 'should update a page' do
-    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
     @storage.add p
     p = @storage.get p
     p.code.should be == 301
@@ -39,15 +38,15 @@ describe Polipus::Storage::S3Store do
   end
 
   it 'should iterate over stored pages' do
-    10.times {|i| @storage.add page_factory("http://www.google.com/p_#{i}", :code => 200, :body => "<html>#{i}</html>")}
+    10.times { |i| @storage.add page_factory("http://www.google.com/p_#{i}", code: 200, body: "<html>#{i}</html>") }
     @storage.count.should be 10
-    @storage.each do |k, page|
+    @storage.each do |k, _page|
       k.should be =~ /[a-f0-9]{32}/
     end
   end
 
   it 'should delete a page' do
-    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
     @storage.add p
     @storage.remove p
     @storage.get(p).should be_nil
@@ -55,8 +54,8 @@ describe Polipus::Storage::S3Store do
   end
 
   it 'should store a page removing a query string from the uuid generation' do
-    p = page_factory 'http://www.asd.com/?asd=lol', :code => 200, :body => '<html></html>'
-    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
     @storage.include_query_string_in_uuid = false
     @storage.add p
     @storage.exists?(p_no_query).should be_true
@@ -64,8 +63,8 @@ describe Polipus::Storage::S3Store do
   end
 
   it 'should store a page removing a query string from the uuid generation no ending slash' do
-    p = page_factory 'http://www.asd.com?asd=lol', :code => 200, :body => '<html></html>'
-    p_no_query = page_factory 'http://www.asd.com', :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
     @storage.include_query_string_in_uuid = false
     @storage.add p
     @storage.exists?(p_no_query).should be_true
@@ -73,7 +72,7 @@ describe Polipus::Storage::S3Store do
   end
 
   it 'should store a page with user data associated' do
-    p = page_factory 'http://www.user.com',  :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
     p.user_data.name = 'Test User Data'
     @storage.add p
     @storage.exists?(p).should be_true
@@ -83,15 +82,15 @@ describe Polipus::Storage::S3Store do
   end
 
   it 'should honor the except parameters' do
-    storage = storage = Polipus::Storage.s3_store(
-        '_test_pages', 
-         {
-          :access_key_id => 'XXXXXXX', 
-          :secret_access_key => 'XXXX'
+    storage = Polipus::Storage.s3_store(
+        '_test_pages',
+        {
+          access_key_id: 'XXXXXXX',
+          secret_access_key: 'XXXX'
         },
         ['body']
       )
-    p = page_factory 'http://www.user-doo.com',  :code => 200, :body => '<html></html>'
+    p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
     storage.add p
     p = storage.get p
 
@@ -101,15 +100,15 @@ describe Polipus::Storage::S3Store do
 
   it 'should return false if a doc not exists' do
     @storage.include_query_string_in_uuid = false
-    p_other  = page_factory 'http://www.asdrrrr.com', :code => 200, :body => '<html></html>'
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
     @storage.exists?(p_other).should be_false
     @storage.add p_other
     @storage.exists?(p_other).should be_true
-    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', :code => 200, :body => '<html></html>'
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
     @storage.exists?(p_other).should be_true
     @storage.include_query_string_in_uuid = true
     @storage.exists?(p_other).should be_false
     @storage.remove p_other
   end
 
-end
+end