polipus 0.3.7 → 0.4.0

data/lib/polipus/version.rb

@@ -1,5 +1,5 @@
 # encoding: UTF-8
 module Polipus
-  VERSION = '0.3.7'
+  VERSION = '0.4.0'
   HOMEPAGE = 'https://github.com/taganaka/polipus'
 end

data/polipus.gemspec

@@ -13,6 +13,7 @@ Gem::Specification.new do |s|
     An easy to use distributed web-crawler framework based on Redis
   )
   s.licenses    = ['MIT']
+  s.platform    = Gem::Platform::RUBY
 
   s.rubyforge_project = 'polipus'
 
@@ -21,26 +22,23 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_paths = ['lib']
 
-  s.add_runtime_dependency 'redis-bloomfilter', '~> 0.0', '>= 0.0.1'
-  s.add_runtime_dependency 'redis-queue', '~> 0.0', '>= 0.0.3'
   s.add_runtime_dependency 'nokogiri', '~> 1.6', '>= 1.6.0'
-  s.add_runtime_dependency 'hiredis', '~> 0.4', '>= 0.4.5'
-  s.add_runtime_dependency 'redis', '~> 3.0', '>= 3.0.4'
-  s.add_runtime_dependency 'mongo', '~> 1.9.0', '>= 1.9.2'
-
-  if defined?(JRUBY_VERSION)
-    s.add_runtime_dependency 'bson', '~> 1.9', '>= 1.9.2'
-  else
-    s.add_runtime_dependency 'bson_ext', '~> 1.9', '>= 1.9.2'
-  end
-  s.add_runtime_dependency 'aws-s3', '~> 0.6', '>= 0.6.3'
   s.add_runtime_dependency 'http-cookie', '~> 1.0', '>= 1.0.1'
 
-  s.add_development_dependency 'rspec', '~> 2.14', '>= 2.14.1'
-  s.add_development_dependency 'vcr', '~> 2.5', '>= 2.5.0'
-  s.add_development_dependency 'webmock', '>= 1.8.0', '< 1.12'
-  s.add_development_dependency 'flexmock', '~> 1.3', '>= 1.3.2'
-  s.add_development_dependency 'rake', '~> 10.3', '>= 10.3.2'
-  s.add_development_dependency 'coveralls'
+  s.add_runtime_dependency 'redis', '~> 3.0', '>= 3.0.4'
+  s.add_runtime_dependency 'hiredis', '~> 0.5', '>= 0.4.5'
+  s.add_runtime_dependency 'redis-queue', '~> 0.0', '>= 0.0.4'
+  s.add_runtime_dependency 'redis-bloomfilter', '~> 0.0', '>= 0.0.3'
+
+  s.add_development_dependency 'mongo', '~>1.11.0'
+  s.add_development_dependency 'rethinkdb', '~>1.15.0'
 
+  s.add_development_dependency 'rake', '~> 10.3'
+  s.add_development_dependency 'rspec', '~> 3.1.0'
+  s.add_development_dependency 'flexmock', '~> 1.3'
+
+  s.add_development_dependency 'vcr', '~> 2.9.0'
+  s.add_development_dependency 'webmock', '~> 1.20.0'
+
+  s.add_development_dependency 'coveralls'
 end

data/spec/{http_spec.rb → polipus/http_spec.rb}

@@ -5,118 +5,107 @@ require 'polipus/http'
 require 'polipus/page'
 
 describe Polipus::HTTP do
-
   it 'should download a page' do
     VCR.use_cassette('http_test') do
       http = Polipus::HTTP.new
       page = http.fetch_page('http://sfbay.craigslist.org/apa/')
-      page.should be_an_instance_of(Polipus::Page)
-      page.doc.search('title').text.strip.should eq 'SF bay area apts/housing for rent classifieds  - craigslist'
-      page.fetched_at.should_not be_nil
-      page.fetched?.should be_true
+      expect(page).to be_an_instance_of(Polipus::Page)
+      expect(page.doc.search('title').text.strip).to eq 'SF bay area apts/housing for rent classifieds  - craigslist'
+      expect(page.fetched_at).not_to be_nil
+      expect(page.fetched?).to be_truthy
     end
   end
 
   it 'should follow a redirect' do
     VCR.use_cassette('http_test_redirect') do
-
       http = Polipus::HTTP.new
       page = http.fetch_page('http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis')
 
-      page.should be_an_instance_of(Polipus::Page)
-      page.code.should be 200
-      page.url.to_s.should eq 'http://greenbytes.de/tech/tc/httpredirects/300.txt'
-      page.body.strip.should eq "You have reached the target\r\nof a 300 redirect."
+      expect(page).to be_an_instance_of(Polipus::Page)
+      expect(page.code).to be 200
+      expect(page.url.to_s).to eq 'http://greenbytes.de/tech/tc/httpredirects/300.txt'
+      expect(page.body.strip).to eq "You have reached the target\r\nof a 300 redirect."
     end
   end
 
   describe 'proxy settings' do
-
     it 'should set proxy correctly using a procedure' do
       http = Polipus::HTTP.new(proxy_host: -> _con { '127.0.0.0' }, proxy_port: -> _con { 8080 })
-      http.proxy_host.should eq '127.0.0.0'
-      http.proxy_port.should be 8080
+      expect(http.proxy_host).to eq '127.0.0.0'
+      expect(http.proxy_port).to be 8080
     end
 
     it 'should set proxy correctly using shorthand method' do
       http = Polipus::HTTP.new(proxy_host_port: -> _con { ['127.0.0.0', 8080] })
-      http.proxy_host_port.should eq ['127.0.0.0', 8080]
+      expect(http.proxy_host_port).to eq ['127.0.0.0', 8080]
     end
 
     it 'should set proxy w/ auth correctly using shorthand method' do
       http = Polipus::HTTP.new(proxy_host_port: -> _con { ['127.0.0.0', 8080, 'a', 'b'] })
-      http.proxy_host_port.should eq ['127.0.0.0', 8080, 'a', 'b']
+      expect(http.proxy_host_port).to eq ['127.0.0.0', 8080, 'a', 'b']
     end
 
     it 'should set proxy settings' do
       http = Polipus::HTTP.new(proxy_host: '127.0.0.0', proxy_port:  8080, proxy_user: 'a', proxy_pass: 'b')
-      http.proxy_port.should be 8080
-      http.proxy_host.should eq '127.0.0.0'
-      http.proxy_user.should eq 'a'
-      http.proxy_pass.should eq 'b'
+      expect(http.proxy_port).to be 8080
+      expect(http.proxy_host).to eq '127.0.0.0'
+      expect(http.proxy_user).to eq 'a'
+      expect(http.proxy_pass).to eq 'b'
     end
-
   end
 
   describe 'compressed content handling' do
-
     it 'should decode gzip content' do
       VCR.use_cassette('gzipped_on') do
         http = Polipus::HTTP.new(logger: Logger.new(STDOUT))
         page = http.fetch_page('http://www.whatsmyip.org/http-compression-test/')
-        page.doc.css('.gzip_yes').should_not be_empty
+        expect(page.doc.css('.gzip_yes')).not_to be_empty
       end
     end
 
     it 'should decode deflate content' do
       http = Polipus::HTTP.new(logger: Logger.new(STDOUT))
       page = http.fetch_page('http://david.fullrecall.com/browser-http-compression-test?compression=deflate-http')
-      page.headers.fetch('content-encoding').first.should eq 'deflate'
-      page.body.include?('deflate-http').should be_true
+      expect(page.headers.fetch('content-encoding').first).to eq 'deflate'
+      expect(page.body.include?('deflate-http')).to be_truthy
     end
-
   end
 
   describe 'staled connections' do
-
     it 'should refresh a staled connection' do
       VCR.use_cassette('http_tconnection_max_hits') do
         http = Polipus::HTTP.new(connection_max_hits: 1, logger: Logger.new(STDOUT))
         http.class.__send__(:attr_reader, :connections)
         http.class.__send__(:attr_reader, :connections_hits)
         http.fetch_page('https://www.yahoo.com/')
-        http.connections['www.yahoo.com'][443].should_not be_nil
+        expect(http.connections['www.yahoo.com'][443]).not_to be_nil
         old_conn = http.connections['www.yahoo.com'][443]
-        http.connections_hits['www.yahoo.com'][443].should be 1
+        expect(http.connections_hits['www.yahoo.com'][443]).to be 1
 
         http.fetch_page('https://www.yahoo.com/tech/expectant-parents-asked-the-internet-to-name-their-83416450388.html')
-        http.connections_hits['www.yahoo.com'][443].should be 1
-        http.connections['www.yahoo.com'][443].should_not be old_conn
+        expect(http.connections_hits['www.yahoo.com'][443]).to be 1
+        expect(http.connections['www.yahoo.com'][443]).not_to be old_conn
       end
     end
-
   end
 
   describe 'cookies' do
-
     it 'should handle cookies correctly' do
       VCR.use_cassette('http_cookies') do
         http = Polipus::HTTP.new(accept_cookies: true)
         http.fetch_page 'http://www.whatarecookies.com/cookietest.asp'
-        http.accept_cookies?.should be_true
-        http.cookie_jar.cookies(URI('http://www.whatarecookies.com/cookietest.asp')).should_not be_empty
+        expect(http.accept_cookies?).to be_truthy
+        expect(http.cookie_jar.cookies(URI('http://www.whatarecookies.com/cookietest.asp'))).not_to be_empty
       end
     end
-
   end
 
   describe 'net errors' do
     it 'should handle net errors correctly' do
       VCR.use_cassette('http_errors') do
         http = Polipus::HTTP.new(open_timeout: 1, read_timeout: 1)
-        http.fetch_page('http://www.wrong-domain.lol/').error.should_not be_nil
+        expect(http.fetch_page('http://www.wrong-domain.lol/').error).not_to be_nil
       end
     end
   end
-
 end

data/spec/{page_spec.rb → polipus/page_spec.rb}

@@ -23,11 +23,11 @@ EOF
   end
 
   it 'should be fetched' do
-    page.fetched?.should be_true
+    expect(page.fetched?).to be_truthy
   end
 
   it 'should honor domain_aliases attribute' do
-    page.links.count.should be 4
+    expect(page.links.count).to be 4
   end
 
   context 'page expiring' do
@@ -41,32 +41,28 @@ EOF
     end
 
     it 'should be marked at expired' do
-      page.expired?(20).should be_true
+      expect(page.expired?(20)).to be_truthy
     end
 
     it 'should NOT be marked at expired' do
-      page.expired?(60).should be_false
+      expect(page.expired?(60)).to be_falsey
     end
   end
 
   context 'page error' do
-
     let(:page) do
       Polipus::Page.new 'http://www.google.com/', error: 'an error'
     end
 
     it 'should serialize an error' do
-      page.to_hash['error'].should eq 'an error'
+      expect(page.to_hash['error']).to eq 'an error'
     end
-
   end
 
   context 'page code' do
     it 'should identify HTTPSuccess code' do
-      Polipus::Page.new('http://www.google.com/', code: 201).success?.should be_true
-      Polipus::Page.new('http://www.google.com/', code: 404).success?.should be_false
+      expect(Polipus::Page.new('http://www.google.com/', code: 201).success?).to be_truthy
+      expect(Polipus::Page.new('http://www.google.com/', code: 404).success?).to be_falsey
     end
-
   end
-
 end

data/spec/{queue_overflow_manager_spec.rb → polipus/queue_overflow/manager_spec.rb}

@@ -33,47 +33,42 @@ describe Polipus::QueueOverflow::Manager do
   end
 
   it 'should remove 10 items' do
-    @manager.perform.should be == [0, 0]
+    expect(@manager.perform).to eq([0, 0])
     20.times { |i| @redis_q << page_factory("http://www.user-doo.com/page_#{i}",  code: 200, body: '<html></html>').to_json  }
-    @manager.perform.should be == [10, 0]
-    @queue_overflow.size.should be == 10
-    @redis_q.size.should be == 10
+    expect(@manager.perform).to eq([10, 0])
+    expect(@queue_overflow.size).to eq(10)
+    expect(@redis_q.size).to eq(10)
   end
 
   it 'should restore 10 items' do
-    @manager.perform.should be == [0, 0]
+    expect(@manager.perform).to eq([0, 0])
     10.times { |i| @queue_overflow << page_factory("http://www.user-doo-bla.com/page_#{i}",  code: 200, body: '<html></html>').to_json }
-    @manager.perform.should be == [0, 10]
-    @queue_overflow.size.should be == 0
-    @redis_q.size.should be == 10
-    @manager.perform.should be == [0, 0]
-
+    expect(@manager.perform).to eq([0, 10])
+    expect(@queue_overflow.size).to eq(0)
+    expect(@redis_q.size).to eq(10)
+    expect(@manager.perform).to eq([0, 0])
   end
 
   it 'should restore 3 items' do
-
-    @manager.perform.should be == [0, 0]
+    expect(@manager.perform).to eq([0, 0])
     3.times { |i| @queue_overflow << page_factory("http://www.user-doo-bu.com/page_#{i}",  code: 200, body: '<html></html>').to_json }
-    @manager.perform.should be == [0, 3]
-    @queue_overflow.size.should be == 0
-    @redis_q.size.should be == 3
-    @manager.perform.should be == [0, 0]
-
+    expect(@manager.perform).to eq([0, 3])
+    expect(@queue_overflow.size).to eq(0)
+    expect(@redis_q.size).to eq(3)
+    expect(@manager.perform).to eq([0, 0])
   end
 
   it 'should restore 0 items' do
-
-    @manager.perform.should be == [0, 0]
+    expect(@manager.perform).to eq([0, 0])
     10.times do|i|
       p = page_factory("http://www.user-doo-bu.com/page_#{i}",  code: 200, body: '<html></html>')
       @storage.add p
       @queue_overflow << p.to_json
     end
-    @manager.perform.should be == [0, 0]
-    @queue_overflow.size.should be == 0
-    @redis_q.size.should be == 0
-    @manager.perform.should be == [0, 0]
-
+    expect(@manager.perform).to eq([0, 0])
+    expect(@queue_overflow.size).to eq(0)
+    expect(@redis_q.size).to eq(0)
+    expect(@manager.perform).to eq([0, 0])
   end
 
   it 'should filter an url based on the spec' do
@@ -83,13 +78,11 @@ describe Polipus::QueueOverflow::Manager do
     @manager.url_filter do |page|
       page.url.to_s.end_with?('page_0') ? false : true
     end
-    @manager.perform.should be == [0, 9]
-    @queue_overflow.size.should be == 0
-    @redis_q.size.should be == 9
+    expect(@manager.perform).to eq([0, 9])
+    expect(@queue_overflow.size).to eq(0)
+    expect(@redis_q.size).to eq(9)
     @manager.url_filter do |_page|
       true
     end
-
   end
-
 end

data/spec/{queue_overflow_spec.rb → polipus/queue_overflow_spec.rb}

@@ -3,12 +3,10 @@ require 'spec_helper'
 require 'polipus/queue_overflow'
 
 describe Polipus::QueueOverflow do
-
   before(:all) do
     @queue_overflow        = Polipus::QueueOverflow.mongo_queue(nil, 'queue_test')
     @queue_overflow_capped = Polipus::QueueOverflow.mongo_queue_capped(nil, 'queue_test_c', max: 20)
     @queue_overflow_uniq   = Polipus::QueueOverflow.mongo_queue(nil, 'queue_test_u', ensure_uniq: true)
-
   end
 
   before(:each) do
@@ -25,26 +23,24 @@ describe Polipus::QueueOverflow do
 
   it 'should work' do
     [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
-      q.empty?.should be_true
-      q.pop.should be_nil
+      expect(q.empty?).to be_truthy
+      expect(q.pop).to be_nil
       q << 'test'
-      q.size.should be == 1
-      q.pop.should be == 'test'
-      q.empty?.should be_true
-      q.pop.should be_nil
-      q.size.should be == 0
-      q.empty?.should be_true
+      expect(q.size).to eq(1)
+      expect(q.pop).to eq('test')
+      expect(q.empty?).to be_truthy
+      expect(q.pop).to be_nil
+      expect(q.size).to eq(0)
+      expect(q.empty?).to be_truthy
     end
-
   end
 
   it 'should act as a queue' do
     [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
       10.times { |i| q << "message_#{i}" }
-      q.size.should be == 10
-      q.pop.should be == 'message_0'
+      expect(q.size).to eq(10)
+      expect(q.pop).to eq('message_0')
     end
-
   end
 
   it 'should work with complex paylod' do
@@ -52,21 +48,19 @@ describe Polipus::QueueOverflow do
       a = { 'a' => [1, 2, 3], 'b' => 'a_string' }
       q << a.to_json
       b = q.pop
-      JSON.parse(b).should be == a
+      expect(JSON.parse(b)).to eq(a)
     end
-
   end
 
   it 'should honor max items if it is capped' do
     30.times { |i| @queue_overflow_capped << "message_#{i}" }
-    @queue_overflow_capped.size.should be == 20
-    @queue_overflow_capped.pop.should be == 'message_10'
+    expect(@queue_overflow_capped.size).to eq(20)
+    expect(@queue_overflow_capped.pop).to eq('message_10')
   end
 
   it 'should contains only unique items' do
     20.times { @queue_overflow_uniq << 'A' }
     20.times { @queue_overflow_uniq << 'B' }
-    @queue_overflow_uniq.size.should be == 2
+    expect(@queue_overflow_uniq.size).to eq(2)
   end
-
 end

data/spec/{robotex_spec.rb → polipus/robotex_spec.rb}

@@ -1,6 +1,6 @@
-# encoding: UTF-8
 require 'spec_helper'
 require 'polipus/robotex'
+
 describe Polipus::Robotex do
   let(:spec_domain) { 'http://www.example.com/' }
   before(:each) do
@@ -19,20 +19,20 @@ Disallow: /locked
 Allow: /locked
 END
     stub_request(:get, 'http://www.example.com/robots.txt')
-    .to_return(body: robots, status: [200, 'OK'], headers: { 'Content-Type' => 'text/plain' })
+      .to_return(body: robots, status: [200, 'OK'], headers: { 'Content-Type' => 'text/plain' })
   end
 
   describe '#initialize' do
     context 'when no arguments are supplied' do
       it 'returns a Robotex with the default user-agent' do
-        Polipus::Robotex.new.user_agent.should == "Robotex/#{Polipus::Robotex::VERSION} (http://www.github.com/chriskite/robotex)"
+        expect(Polipus::Robotex.new.user_agent).to eq("Robotex/#{Polipus::Robotex::VERSION} (http://www.github.com/chriskite/robotex)")
       end
     end
 
     context 'when a user-agent is specified' do
       it 'returns a Robotex with the specified user-agent' do
         ua = 'My User Agent'
-        Polipus::Robotex.new(ua).user_agent.should == ua
+        expect(Polipus::Robotex.new(ua).user_agent).to eq(ua)
       end
     end
   end
@@ -41,28 +41,28 @@ END
     context 'when the robots.txt disallows the user-agent to the url' do
       it 'returns false' do
         robotex = Polipus::Robotex.new('bender')
-        robotex.allowed?(spec_domain + 'my_shiny_metal_ass').should be_false
+        expect(robotex.allowed?(spec_domain + 'my_shiny_metal_ass')).to be_falsey
       end
     end
 
     context 'when the robots.txt disallows the user-agent to some urls, but allows this one' do
       it 'returns true' do
         robotex = Polipus::Robotex.new('bender')
-        robotex.allowed?(spec_domain + 'cigars').should be_true
+        expect(robotex.allowed?(spec_domain + 'cigars')).to be_truthy
       end
     end
 
     context 'when the robots.txt disallows any user-agent to the url' do
       it 'returns false' do
         robotex = Polipus::Robotex.new
-        robotex.allowed?(spec_domain + 'login').should be_false
+        expect(robotex.allowed?(spec_domain + 'login')).to be_falsey
       end
     end
 
     context 'when the robots.txt disallows and then allows the url' do
       it 'returns false' do
         robotex = Polipus::Robotex.new
-        robotex.allowed?(spec_domain + 'locked').should be_false
+        expect(robotex.allowed?(spec_domain + 'locked')).to be_falsey
       end
     end
   end
@@ -71,16 +71,15 @@ END
     context 'when no Crawl-Delay is specified for the user-agent' do
       it 'returns nil' do
         robotex = Polipus::Robotex.new
-        robotex.delay(spec_domain).should be_nil
+        expect(robotex.delay(spec_domain)).to be_nil
       end
 
       context 'when Crawl-Delay is specified for the user-agent' do
         it 'returns the delay as a Fixnum' do
           robotex = Polipus::Robotex.new('msnbot')
-          robotex.delay(spec_domain).should == 20
+          expect(robotex.delay(spec_domain)).to eq(20)
         end
       end
     end
   end
-
 end