polipus 0.3.0 → 0.3.1

data/lib/polipus/version.rb

@@ -1,4 +1,4 @@
 module Polipus
-  VERSION = "0.3.0"
-  HOMEPAGE = "https://github.com/taganaka/polipus"
-end
+  VERSION = '0.3.1'
+  HOMEPAGE = 'https://github.com/taganaka/polipus'
+end

data/polipus.gemspec

@@ -1,25 +1,25 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "polipus/version"
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'polipus/version'
 
 Gem::Specification.new do |s|
-  s.name        = "polipus"
+  s.name        = 'polipus'
   s.version     = Polipus::VERSION
-  s.authors     = ["Francesco Laurita"]
-  s.email       = ["francesco.laurita@gmail.com"]
+  s.authors     = ['Francesco Laurita']
+  s.email       = ['francesco.laurita@gmail.com']
   s.homepage    = Polipus::HOMEPAGE
-  s.summary     = %q{Polipus distributed web-crawler framework}
-  s.description = %q{
+  s.summary     = %q(Polipus distributed web-crawler framework)
+  s.description = %q(
     An easy to use distributed web-crawler framework based on Redis
-  }
-  s.licenses    = ["MIT"]
+  )
+  s.licenses    = ['MIT']
 
-  s.rubyforge_project = "polipus"
+  s.rubyforge_project = 'polipus'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_paths = ['lib']
 
   s.add_runtime_dependency 'redis-bloomfilter', '~> 0.0', '>= 0.0.1'
   s.add_runtime_dependency 'redis-queue', '~> 0.0', '>= 0.0.3'
@@ -43,5 +43,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rake', '~> 10.3', '>= 10.3.2'
   s.add_development_dependency 'coveralls'
 
-
 end

data/spec/clear.rb

@@ -1,4 +1,4 @@
-require "yaml"
+require 'yaml'
 Dir.glob('./cassettes/*.yml').each do|f|
   next unless f =~ /[a-f0-9]{32}/
   d = YAML.load_file(f)
@@ -6,6 +6,6 @@ Dir.glob('./cassettes/*.yml').each do|f|
     r['request'].delete('headers')
     r['response'].delete('headers')
   end
-  File.open(f, 'w') {|fw| fw.write(d.to_yaml) }
-  #puts d.to_yaml
+  File.open(f, 'w') { |fw| fw.write(d.to_yaml) }
+  # puts d.to_yaml
 end

data/spec/http_spec.rb

@@ -1,16 +1,16 @@
-require "spec_helper"
-require "mongo"
-require "polipus/http"
-require "polipus/page"
+require 'spec_helper'
+require 'mongo'
+require 'polipus/http'
+require 'polipus/page'
 
 describe Polipus::HTTP do
-  
+
   it 'should download a page' do
     VCR.use_cassette('http_test') do
       http = Polipus::HTTP.new
-      page = http.fetch_page("http://sfbay.craigslist.org/apa/")
+      page = http.fetch_page('http://sfbay.craigslist.org/apa/')
       page.should be_an_instance_of(Polipus::Page)
-      page.doc.search("title").text.strip.should eq "SF bay area apts/housing for rent classifieds  - craigslist"
+      page.doc.search('title').text.strip.should eq 'SF bay area apts/housing for rent classifieds  - craigslist'
       page.fetched_at.should_not be_nil
       page.fetched?.should be_true
     end
@@ -20,11 +20,11 @@ describe Polipus::HTTP do
     VCR.use_cassette('http_test_redirect') do
 
       http = Polipus::HTTP.new
-      page = http.fetch_page("http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis")
+      page = http.fetch_page('http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis')
 
       page.should be_an_instance_of(Polipus::Page)
       page.code.should be 200
-      page.url.to_s.should eq "http://greenbytes.de/tech/tc/httpredirects/300.txt"
+      page.url.to_s.should eq 'http://greenbytes.de/tech/tc/httpredirects/300.txt'
       page.body.strip.should eq "You have reached the target\r\nof a 300 redirect."
     end
   end
@@ -32,59 +32,58 @@ describe Polipus::HTTP do
   describe 'proxy settings' do
 
     it 'should set proxy correctly using a procedure' do
-      http = Polipus::HTTP.new({proxy_host: -> con { "127.0.0.0" }, proxy_port: -> con { 8080 }})
-      http.proxy_host.should eq "127.0.0.0"
+      http = Polipus::HTTP.new(proxy_host: -> _con { '127.0.0.0' }, proxy_port: -> _con { 8080 })
+      http.proxy_host.should eq '127.0.0.0'
       http.proxy_port.should be 8080
     end
 
     it 'should set proxy correctly using shorthand method' do
-      http = Polipus::HTTP.new({proxy_host_port: -> con {["127.0.0.0", 8080] }})
-      http.proxy_host_port.should eq ["127.0.0.0", 8080]
+      http = Polipus::HTTP.new(proxy_host_port: -> _con { ['127.0.0.0', 8080] })
+      http.proxy_host_port.should eq ['127.0.0.0', 8080]
       http.proxy_port.should be 8080
-      http.proxy_host.should eq "127.0.0.0"
+      http.proxy_host.should eq '127.0.0.0'
     end
 
     it 'should set proxy settings' do
-      http = Polipus::HTTP.new({proxy_host: "127.0.0.0", proxy_port:  8080 })
+      http = Polipus::HTTP.new(proxy_host: '127.0.0.0', proxy_port:  8080)
       http.proxy_port.should be 8080
-      http.proxy_host.should eq "127.0.0.0"
+      http.proxy_host.should eq '127.0.0.0'
     end
 
   end
 
-
   describe 'compressed content handling' do
 
     it 'should decode gzip content' do
       VCR.use_cassette('gzipped_on') do
         http = Polipus::HTTP.new(logger: Logger.new(STDOUT))
-        page = http.fetch_page("http://www.whatsmyip.org/http-compression-test/")
+        page = http.fetch_page('http://www.whatsmyip.org/http-compression-test/')
         page.doc.css('.gzip_yes').should_not be_empty
       end
     end
 
     it 'should decode deflate content' do
       http = Polipus::HTTP.new(logger: Logger.new(STDOUT))
-      page = http.fetch_page("http://david.fullrecall.com/browser-http-compression-test?compression=deflate-http")
+      page = http.fetch_page('http://david.fullrecall.com/browser-http-compression-test?compression=deflate-http')
       page.headers.fetch('content-encoding').first.should eq 'deflate'
-      page.body.include?("deflate-http").should be_true
+      page.body.include?('deflate-http').should be_true
     end
 
   end
 
   describe 'staled connections' do
-    
+
     it 'should refresh a staled connection' do
       VCR.use_cassette('http_tconnection_max_hits') do
         http = Polipus::HTTP.new(connection_max_hits: 1, logger: Logger.new(STDOUT))
         http.class.__send__(:attr_reader, :connections)
         http.class.__send__(:attr_reader, :connections_hits)
-        http.fetch_page("https://www.yahoo.com/")
+        http.fetch_page('https://www.yahoo.com/')
         http.connections['www.yahoo.com'][443].should_not be_nil
         old_conn = http.connections['www.yahoo.com'][443]
         http.connections_hits['www.yahoo.com'][443].should be 1
 
-        http.fetch_page("https://www.yahoo.com/tech/expectant-parents-asked-the-internet-to-name-their-83416450388.html")
+        http.fetch_page('https://www.yahoo.com/tech/expectant-parents-asked-the-internet-to-name-their-83416450388.html')
         http.connections_hits['www.yahoo.com'][443].should be 1
         http.connections['www.yahoo.com'][443].should_not be old_conn
       end
@@ -97,9 +96,9 @@ describe Polipus::HTTP do
     it 'should handle cookies correctly' do
       VCR.use_cassette('http_cookies') do
         http = Polipus::HTTP.new(accept_cookies: true)
-        http.fetch_page "http://www.whatarecookies.com/cookietest.asp"
+        http.fetch_page 'http://www.whatarecookies.com/cookietest.asp'
         http.accept_cookies?.should be_true
-        http.cookie_jar.cookies(URI("http://www.whatarecookies.com/cookietest.asp")).should_not be_empty
+        http.cookie_jar.cookies(URI('http://www.whatarecookies.com/cookietest.asp')).should_not be_empty
       end
     end
 
@@ -108,10 +107,10 @@ describe Polipus::HTTP do
   describe 'net errors' do
     it 'should handle net errors correctly' do
       VCR.use_cassette('http_errors') do
-        http = Polipus::HTTP.new(open_timeout:1, read_timeout: 1)
-        http.fetch_page("http://www.wrong-domain.lol/").error.should_not be_nil
+        http = Polipus::HTTP.new(open_timeout: 1, read_timeout: 1)
+        http.fetch_page('http://www.wrong-domain.lol/').error.should_not be_nil
       end
     end
   end
 
-end
+end

data/spec/page_spec.rb

@@ -1,5 +1,5 @@
-require "spec_helper"
-require "polipus/page"
+require 'spec_helper'
+require 'polipus/page'
 
 describe Polipus::Page do
   let(:page) do
@@ -14,29 +14,29 @@ describe Polipus::Page do
       </body>
     </html>
 EOF
-    Polipus::Page.new 'http://www.google.com/', 
-      code: 200, 
-      body: body, 
-      headers: {'content-type' => ['text/html']}, 
-      domain_aliases: %w(www.google.com google.com)
+    Polipus::Page.new 'http://www.google.com/',
+                      code: 200,
+                      body: body,
+                      headers: { 'content-type' => ['text/html'] },
+                      domain_aliases: %w(www.google.com google.com)
   end
 
   it 'should be fetched' do
     page.fetched?.should be_true
   end
-  
+
   it 'should honor domain_aliases attribute' do
     page.links.count.should be 4
   end
 
   context 'page expiring' do
     let(:page) do
-      Polipus::Page.new 'http://www.google.com/', 
-        code: 200, 
-        body: '', 
-        headers: {'content-type' => ['text/html']}, 
-        domain_aliases: %w(www.google.com google.com),
-        fetched_at: (Time.now.to_i - 30)
+      Polipus::Page.new 'http://www.google.com/',
+                        code: 200,
+                        body: '',
+                        headers: { 'content-type' => ['text/html'] },
+                        domain_aliases: %w(www.google.com google.com),
+                        fetched_at: (Time.now.to_i - 30)
     end
 
     it 'should be marked at expired' do
@@ -49,12 +49,12 @@ EOF
   end
 
   context 'page error' do
-    
+
     let(:page) do
       Polipus::Page.new 'http://www.google.com/', error: 'an error'
     end
 
-    it 'should serialize an error' do 
+    it 'should serialize an error' do
       page.to_hash['error'].should eq 'an error'
     end
 

data/spec/polipus_spec.rb

@@ -1,11 +1,11 @@
-require "spec_helper"
+require 'spec_helper'
 
 describe Polipus::PolipusCrawler do
-  after(:each) {Redis.new(db:10).flushdb}
-  let(:p_options) {
+  after(:each) { Redis.new(db: 10).flushdb }
+  let(:p_options) do
     {
       workers: 1,
-      redis_options: {host: 'localhost', db:10},
+      redis_options: { host: 'localhost', db: 10 },
       depth_limit: 1,
       queue_timeout: 1,
       user_agent: 'polipus-rspec',
@@ -13,55 +13,58 @@ describe Polipus::PolipusCrawler do
       logger_level: Logger::DEBUG,
       storage: Polipus::Storage.memory_store
     }
-  }
-  let(:polipus) {
-    Polipus::PolipusCrawler.new("polipus-rspec", ["http://rubygems.org/gems"], p_options)
-  }
+  end
+  let(:polipus) do
+    Polipus::PolipusCrawler.new('polipus-rspec', ['http://rubygems.org/gems'], p_options)
+  end
 
-  let(:init_page){
-    Polipus::Page.new "http://rubygems.org/gems"
-  }
+  let(:init_page)do
+    Polipus::Page.new 'http://rubygems.org/gems'
+  end
 
-  let(:logger){Logger.new(nil)}
+  let(:logger) { Logger.new(nil) }
 
-  context "polipus" do
+  context 'polipus' do
 
-    it "should create a polipus instance" do
+    it 'should create a polipus instance' do
       polipus.should be_an_instance_of Polipus::PolipusCrawler
     end
 
-    it "should execute a crawling session" do
+    it 'should execute a crawling session' do
       polipus.takeover
       polipus.storage.exists?(init_page).should be_true
       polipus.storage.get(init_page).links.count.should be polipus.storage.count
     end
 
-    it "should filter unwanted urls" do
+    it 'should filter unwanted urls' do
       polipus.skip_links_like(/\/pages\//)
       polipus.takeover
       polipus.storage.get(init_page).links
         .reject { |e| e.path.to_s =~ /\/pages\// }.count.should be polipus.storage.count
     end
 
-    it "should follow only wanted urls" do
+    it 'should follow only wanted urls' do
       polipus.follow_links_like(/\/pages\//)
       polipus.follow_links_like(/\/gems$/)
       polipus.takeover
       polipus.storage.get(init_page).links
-        .reject { |e| ![/\/pages\//, /\/gems$/].any?{|p| e.path =~ p} }
+        .reject { |e| ![/\/pages\//, /\/gems$/].any? { |p| e.path =~ p } }
         .count.should be polipus.storage.count
     end
 
-    it "should refresh expired pages" do
+    it 'should refresh expired pages' do
       polipus.ttl_page = 3600
       polipus.takeover
-      polipus.storage.each {|id, page| page.fetched_at = page.fetched_at - 3600; polipus.storage.add(page)}
-      polipus.storage.each {|id, page| page.expired?(3600).should be_true}
+      polipus.storage.each do |_id, page|
+        page.fetched_at = page.fetched_at - 3600
+        polipus.storage.add(page)
+      end
+      polipus.storage.each { |_id, page| page.expired?(3600).should be_true }
       polipus.takeover
-      polipus.storage.each {|id, page| page.expired?(3600).should be_false}
+      polipus.storage.each { |_id, page| page.expired?(3600).should be_false }
     end
 
-    it "should re-download seeder urls no matter what" do
+    it 'should re-download seeder urls no matter what' do
       cache_hit = {}
       polipus.follow_links_like(/\/gems$/)
       polipus.on_page_downloaded do |page|
@@ -70,26 +73,26 @@ describe Polipus::PolipusCrawler do
       end
       polipus.takeover
       polipus.takeover
-      cache_hit["http://rubygems.org/gems"].should be 2
+      cache_hit['http://rubygems.org/gems'].should be 2
     end
 
-    it "should call on_page_error code blocks when a page has error" do
-      p = Polipus::PolipusCrawler.new("polipus-rspec", ["http://dasd.adad.dom/"], p_options.merge(open_timeout:1, read_timeout: 1))
+    it 'should call on_page_error code blocks when a page has error' do
+      p = Polipus::PolipusCrawler.new('polipus-rspec', ['http://dasd.adad.dom/'], p_options.merge(open_timeout: 1, read_timeout: 1))
       a_page = nil
-      p.on_page_error {|page| a_page = page}
+      p.on_page_error { |page| a_page = page }
       p.takeover
       a_page.should_not be_nil
       a_page.error.should_not be_nil
     end
 
-    it "should obey to the robots.txt file" do
+    it 'should obey to the robots.txt file' do
       lopt = p_options
       lopt[:obey_robots_txt] = true
-      polipus = Polipus::PolipusCrawler.new("polipus-rspec", ["https://rubygems.org/gems/polipus"], lopt)
+      polipus = Polipus::PolipusCrawler.new('polipus-rspec', ['https://rubygems.org/gems/polipus'], lopt)
       polipus.depth_limit = 1
       polipus.takeover
-      polipus.storage.each {|id, page| (page.url.path =~ /$\/downloads\//).should be_false}
+      polipus.storage.each { |_id, page| (page.url.path =~ /$\/downloads\//).should be_false }
     end
 
   end
-end
+end

data/spec/queue_overflow_manager_spec.rb

@@ -1,20 +1,22 @@
-require "spec_helper"
-require "polipus/queue_overflow"
-require "redis-queue"
+require 'spec_helper'
+require 'mongo'
+require 'polipus/queue_overflow'
+require 'redis-queue'
 
 describe Polipus::QueueOverflow::Manager do
   before(:all) do
-    @mongo = Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('_test_polipus')
+    @mongo = Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('_test_polipus')
     @mongo['_test_pages'].drop
     @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
-    @redis_q = Redis::Queue.new("queue_test","bp_queue_test", :redis => Redis.new())
-    @queue_overflow   = Polipus::QueueOverflow.mongo_queue(nil, "queue_test")
+    @redis_q = Redis::Queue.new('queue_test', 'bp_queue_test', redis: Redis.new)
+    @queue_overflow   = Polipus::QueueOverflow.mongo_queue(nil, 'queue_test')
     @redis = Redis.new
-    @polipus = flexmock("polipus")
+    @polipus = flexmock('polipus')
     @polipus.should_receive(:queue_overflow_adapter).and_return(@queue_overflow)
     @polipus.should_receive(:storage).and_return(@storage)
     @polipus.should_receive(:redis).and_return(@redis)
-    @polipus.should_receive(:job_name).and_return("___test")
+    @polipus.should_receive(:job_name).and_return('___test')
+    @polipus.should_receive(:logger).and_return(Logger.new(nil))
     @manager = Polipus::QueueOverflow::Manager.new(@polipus, @redis_q, 10)
   end
 
@@ -30,16 +32,16 @@ describe Polipus::QueueOverflow::Manager do
   end
 
   it 'should remove 10 items' do
-    @manager.perform.should be == [0,0]
-    20.times {|i| @redis_q << page_factory("http://www.user-doo.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json  }
+    @manager.perform.should be == [0, 0]
+    20.times { |i| @redis_q << page_factory("http://www.user-doo.com/page_#{i}",  code: 200, body: '<html></html>').to_json  }
     @manager.perform.should be == [10, 0]
     @queue_overflow.size.should be == 10
     @redis_q.size.should be == 10
   end
 
   it 'should restore 10 items' do
-    @manager.perform.should be == [0,0]
-    10.times {|i| @queue_overflow << page_factory("http://www.user-doo-bla.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json }
+    @manager.perform.should be == [0, 0]
+    10.times { |i| @queue_overflow << page_factory("http://www.user-doo-bla.com/page_#{i}",  code: 200, body: '<html></html>').to_json }
     @manager.perform.should be == [0, 10]
     @queue_overflow.size.should be == 0
     @redis_q.size.should be == 10
@@ -48,45 +50,45 @@ describe Polipus::QueueOverflow::Manager do
   end
 
   it 'should restore 3 items' do
-    
-    @manager.perform.should be == [0,0]
-    3.times {|i| @queue_overflow << page_factory("http://www.user-doo-bu.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json }
+
+    @manager.perform.should be == [0, 0]
+    3.times { |i| @queue_overflow << page_factory("http://www.user-doo-bu.com/page_#{i}",  code: 200, body: '<html></html>').to_json }
     @manager.perform.should be == [0, 3]
     @queue_overflow.size.should be == 0
     @redis_q.size.should be == 3
     @manager.perform.should be == [0, 0]
-    
+
   end
 
   it 'should restore 0 items' do
-    
-    @manager.perform.should be == [0,0]
-    10.times {|i|
-      p = page_factory("http://www.user-doo-bu.com/page_#{i}",  :code => 200, :body => '<html></html>')
+
+    @manager.perform.should be == [0, 0]
+    10.times do|i|
+      p = page_factory("http://www.user-doo-bu.com/page_#{i}",  code: 200, body: '<html></html>')
       @storage.add p
-      @queue_overflow << p.to_json 
-    }
+      @queue_overflow << p.to_json
+    end
     @manager.perform.should be == [0, 0]
     @queue_overflow.size.should be == 0
     @redis_q.size.should be == 0
     @manager.perform.should be == [0, 0]
-    
+
   end
 
   it 'should filter an url based on the spec' do
     @queue_overflow.clear
     @redis_q.clear
-    10.times {|i| @queue_overflow << page_factory("http://www.user-doo.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json  }
+    10.times { |i| @queue_overflow << page_factory("http://www.user-doo.com/page_#{i}",  code: 200, body: '<html></html>').to_json  }
     @manager.url_filter do |page|
-      page.url.to_s.end_with?("page_0") ? false : true
+      page.url.to_s.end_with?('page_0') ? false : true
     end
-    @manager.perform.should be == [0,9]
+    @manager.perform.should be == [0, 9]
     @queue_overflow.size.should be == 0
     @redis_q.size.should be == 9
-    @manager.url_filter do |page|
+    @manager.url_filter do |_page|
       true
     end
 
   end
 
-end
+end