polipus 0.2.2 → 0.3.0

data/spec/http_spec.rb

@@ -12,6 +12,7 @@ describe Polipus::HTTP do
       page.should be_an_instance_of(Polipus::Page)
       page.doc.search("title").text.strip.should eq "SF bay area apts/housing for rent classifieds  - craigslist"
       page.fetched_at.should_not be_nil
+      page.fetched?.should be_true
     end
   end
 
@@ -52,16 +53,22 @@ describe Polipus::HTTP do
   end
 
 
-  describe 'gzipped content handling' do
+  describe 'compressed content handling' do
 
     it 'should decode gzip content' do
       VCR.use_cassette('gzipped_on') do
-        http = Polipus::HTTP.new(gzip_enabled: true, logger: Logger.new(STDOUT))
+        http = Polipus::HTTP.new(logger: Logger.new(STDOUT))
         page = http.fetch_page("http://www.whatsmyip.org/http-compression-test/")
         page.doc.css('.gzip_yes').should_not be_empty
       end
     end
 
+    it 'should decode deflate content' do
+      http = Polipus::HTTP.new(logger: Logger.new(STDOUT))
+      page = http.fetch_page("http://david.fullrecall.com/browser-http-compression-test?compression=deflate-http")
+      page.headers.fetch('content-encoding').first.should eq 'deflate'
+      page.body.include?("deflate-http").should be_true
+    end
 
   end
 
@@ -82,6 +89,29 @@ describe Polipus::HTTP do
         http.connections['www.yahoo.com'][443].should_not be old_conn
       end
     end
+
+  end
+
+  describe 'cookies' do
+
+    it 'should handle cookies correctly' do
+      VCR.use_cassette('http_cookies') do
+        http = Polipus::HTTP.new(accept_cookies: true)
+        http.fetch_page "http://www.whatarecookies.com/cookietest.asp"
+        http.accept_cookies?.should be_true
+        http.cookie_jar.cookies(URI("http://www.whatarecookies.com/cookietest.asp")).should_not be_empty
+      end
+    end
+
+  end
+
+  describe 'net errors' do
+    it 'should handle net errors correctly' do
+      VCR.use_cassette('http_errors') do
+        http = Polipus::HTTP.new(open_timeout:1, read_timeout: 1)
+        http.fetch_page("http://www.wrong-domain.lol/").error.should_not be_nil
+      end
+    end
   end
 
 end

data/spec/page_spec.rb

@@ -48,5 +48,24 @@ EOF
     end
   end
 
+  context 'page error' do
+    
+    let(:page) do
+      Polipus::Page.new 'http://www.google.com/', error: 'an error'
+    end
+
+    it 'should serialize an error' do 
+      page.to_hash['error'].should eq 'an error'
+    end
+
+  end
+
+  context 'page code' do
+    it 'should identify HTTPSuccess code' do
+      Polipus::Page.new('http://www.google.com/', code: 201).success?.should be_true
+      Polipus::Page.new('http://www.google.com/', code: 404).success?.should be_false
+    end
+
+  end
 
 end

data/spec/polipus_spec.rb

@@ -73,5 +73,23 @@ describe Polipus::PolipusCrawler do
       cache_hit["http://rubygems.org/gems"].should be 2
     end
 
+    it "should call on_page_error code blocks when a page has error" do
+      p = Polipus::PolipusCrawler.new("polipus-rspec", ["http://dasd.adad.dom/"], p_options.merge(open_timeout:1, read_timeout: 1))
+      a_page = nil
+      p.on_page_error {|page| a_page = page}
+      p.takeover
+      a_page.should_not be_nil
+      a_page.error.should_not be_nil
+    end
+
+    it "should obey to the robots.txt file" do
+      lopt = p_options
+      lopt[:obey_robots_txt] = true
+      polipus = Polipus::PolipusCrawler.new("polipus-rspec", ["https://rubygems.org/gems/polipus"], lopt)
+      polipus.depth_limit = 1
+      polipus.takeover
+      polipus.storage.each {|id, page| (page.url.path =~ /$\/downloads\//).should be_false}
+    end
+
   end
 end

data/spec/robotex_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+require "polipus/robotex"
+describe Polipus::Robotex do
+  let(:spec_domain){"http://www.example.com/"}
+  before(:each) do
+    robots = <<-END
+User-Agent: msnbot
+Crawl-Delay: 20
+
+User-Agent: bender
+Disallow: /my_shiny_metal_ass
+
+User-Agent: *
+Disallow: /login
+Allow: /
+
+Disallow: /locked
+Allow: /locked
+END
+    stub_request(:get, 'http://www.example.com/robots.txt')
+    .to_return(:body => robots, :status => [200, "OK"], :headers => { "Content-Type" => 'text/plain' })
+  end
+  
+
+  describe '#initialize' do
+    context 'when no arguments are supplied' do
+      it 'returns a Robotex with the default user-agent' do
+        Polipus::Robotex.new.user_agent.should == "Robotex/#{Polipus::Robotex::VERSION} (http://www.github.com/chriskite/robotex)"
+      end
+    end
+
+    context 'when a user-agent is specified' do
+      it 'returns a Robotex with the specified user-agent' do
+        ua = 'My User Agent'
+        Polipus::Robotex.new(ua).user_agent.should == ua
+      end
+    end
+  end
+
+  describe '#allowed?' do
+    context 'when the robots.txt disallows the user-agent to the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new('bender')
+        robotex.allowed?(spec_domain + 'my_shiny_metal_ass').should be_false
+      end
+    end
+
+    context 'when the robots.txt disallows the user-agent to some urls, but allows this one' do
+      it 'returns true' do
+        robotex = Polipus::Robotex.new('bender')
+        robotex.allowed?(spec_domain + 'cigars').should be_true
+      end
+    end
+
+    context 'when the robots.txt disallows any user-agent to the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new
+        robotex.allowed?(spec_domain + 'login').should be_false
+      end
+    end
+
+    context 'when the robots.txt disallows and then allows the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new
+        robotex.allowed?(spec_domain + 'locked').should be_false
+      end
+    end
+  end
+
+  describe '#delay' do
+    context 'when no Crawl-Delay is specified for the user-agent' do
+      it 'returns nil' do
+        robotex = Polipus::Robotex.new
+        robotex.delay(spec_domain).should be_nil 
+      end
+
+    context 'when Crawl-Delay is specified for the user-agent' do
+      it 'returns the delay as a Fixnum' do
+        robotex = Polipus::Robotex.new('msnbot')
+        robotex.delay(spec_domain).should == 20
+      end
+    end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -6,6 +6,7 @@
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 require "digest/md5"
 require "coveralls"
+require "webmock/rspec"
 
 Coveralls.wear!
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polipus
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Francesco Laurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-23 00:00:00.000000000 Z
+date: 2014-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-bloomfilter
@@ -316,14 +316,16 @@ files:
 - .gitignore
 - .rspec
 - .travis.yml
-- AUTHORS
+- AUTHORS.md
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
-- README.rdoc
 - Rakefile
 - examples/basic.rb
+- examples/error_handling.rb
 - examples/incremental.rb
+- examples/robots_txt_handling.rb
 - examples/survival.rb
 - lib/polipus.rb
 - lib/polipus/http.rb
@@ -338,6 +340,7 @@ files:
 - lib/polipus/queue_overflow/manager.rb
 - lib/polipus/queue_overflow/mongo_queue.rb
 - lib/polipus/queue_overflow/mongo_queue_capped.rb
+- lib/polipus/robotex.rb
 - lib/polipus/storage.rb
 - lib/polipus/storage/base.rb
 - lib/polipus/storage/dev_null.rb
@@ -350,6 +353,7 @@ files:
 - lib/polipus/version.rb
 - polipus.gemspec
 - spec/cassettes/08b228db424a926e1ed6ab63b38d847e.yml
+- spec/cassettes/11c3eb8bf35dfc179dc5ce44f6f5f458.yml
 - spec/cassettes/1f6e1d7743ecaa86594b4e68a6462689.yml
 - spec/cassettes/20aa41f181b49f00078c3ca30bad5afe.yml
 - spec/cassettes/4640919145753505af2d0f8423de37f3.yml
@@ -361,10 +365,12 @@ files:
 - spec/cassettes/ab333f89535a2efb284913fede6aa7c7.yml
 - spec/cassettes/ae5d7cffde3f53122cdf79f3d1367e8e.yml
 - spec/cassettes/b389efd1dcb8f09393b5aae1627c2a83.yml
+- spec/cassettes/bc6fb220895689be7eeb05b09969a18d.yml
 - spec/cassettes/c5ce68499027d490adfbb6e5541881e4.yml
 - spec/cassettes/ce16b11a7df0b70fe90c7f90063fdb8c.yml
 - spec/cassettes/ffe3d588b6df4b9de35e5a7ccaf5a81b.yml
 - spec/cassettes/gzipped_on.yml
+- spec/cassettes/http_cookies.yml
 - spec/cassettes/http_tconnection_max_hits.yml
 - spec/cassettes/http_test.yml
 - spec/cassettes/http_test_redirect.yml
@@ -374,6 +380,7 @@ files:
 - spec/polipus_spec.rb
 - spec/queue_overflow_manager_spec.rb
 - spec/queue_overflow_spec.rb
+- spec/robotex_spec.rb
 - spec/spec_helper.rb
 - spec/storage_memory_spec.rb
 - spec/storage_mongo_spec.rb
@@ -405,6 +412,7 @@ specification_version: 4
 summary: Polipus distributed web-crawler framework
 test_files:
 - spec/cassettes/08b228db424a926e1ed6ab63b38d847e.yml
+- spec/cassettes/11c3eb8bf35dfc179dc5ce44f6f5f458.yml
 - spec/cassettes/1f6e1d7743ecaa86594b4e68a6462689.yml
 - spec/cassettes/20aa41f181b49f00078c3ca30bad5afe.yml
 - spec/cassettes/4640919145753505af2d0f8423de37f3.yml
@@ -416,10 +424,12 @@ test_files:
 - spec/cassettes/ab333f89535a2efb284913fede6aa7c7.yml
 - spec/cassettes/ae5d7cffde3f53122cdf79f3d1367e8e.yml
 - spec/cassettes/b389efd1dcb8f09393b5aae1627c2a83.yml
+- spec/cassettes/bc6fb220895689be7eeb05b09969a18d.yml
 - spec/cassettes/c5ce68499027d490adfbb6e5541881e4.yml
 - spec/cassettes/ce16b11a7df0b70fe90c7f90063fdb8c.yml
 - spec/cassettes/ffe3d588b6df4b9de35e5a7ccaf5a81b.yml
 - spec/cassettes/gzipped_on.yml
+- spec/cassettes/http_cookies.yml
 - spec/cassettes/http_tconnection_max_hits.yml
 - spec/cassettes/http_test.yml
 - spec/cassettes/http_test_redirect.yml
@@ -429,6 +439,7 @@ test_files:
 - spec/polipus_spec.rb
 - spec/queue_overflow_manager_spec.rb
 - spec/queue_overflow_spec.rb
+- spec/robotex_spec.rb
 - spec/spec_helper.rb
 - spec/storage_memory_spec.rb
 - spec/storage_mongo_spec.rb

data/AUTHORS

@@ -1,2 +0,0 @@
-Francesco Laurita <francesco.laurita@gmail.com>
-Tobias L. Maier <http://tobiasmaier.info/>

data/README.rdoc

@@ -1,3 +0,0 @@
-= polipus
-
-Visit https://github.com/taganaka/polipus for further details.