RubyGems - parallel588_polipus - Versions diffs - 0.4.0 - Mend

parallel588_polipus 0.4.0

Files changed (72) hide show

checksums.yaml +7 -0
data/.document +5 -0
data/.gitignore +53 -0
data/.rspec +2 -0
data/.rubocop.yml +17 -0
data/.rubocop_todo.yml +33 -0
data/.travis.yml +22 -0
data/AUTHORS.md +5 -0
data/CHANGELOG.md +61 -0
data/Gemfile +12 -0
data/LICENSE.txt +20 -0
data/README.md +70 -0
data/Rakefile +8 -0
data/examples/basic.rb +63 -0
data/examples/error_handling.rb +23 -0
data/examples/incremental.rb +63 -0
data/examples/robots_txt_handling.rb +14 -0
data/examples/survival.rb +10 -0
data/lib/polipus.rb +488 -0
data/lib/polipus/http.rb +282 -0
data/lib/polipus/page.rb +256 -0
data/lib/polipus/plugin.rb +14 -0
data/lib/polipus/plugins/cleaner.rb +25 -0
data/lib/polipus/plugins/sample.rb +15 -0
data/lib/polipus/plugins/sleeper.rb +22 -0
data/lib/polipus/queue_overflow.rb +26 -0
data/lib/polipus/queue_overflow/base.rb +7 -0
data/lib/polipus/queue_overflow/dev_null_queue.rb +34 -0
data/lib/polipus/queue_overflow/manager.rb +57 -0
data/lib/polipus/queue_overflow/mongo_queue.rb +60 -0
data/lib/polipus/queue_overflow/mongo_queue_capped.rb +29 -0
data/lib/polipus/queue_overflow/worker.rb +24 -0
data/lib/polipus/robotex.rb +145 -0
data/lib/polipus/signal_handler.rb +42 -0
data/lib/polipus/storage.rb +31 -0
data/lib/polipus/storage/base.rb +20 -0
data/lib/polipus/storage/dev_null.rb +35 -0
data/lib/polipus/storage/memory_store.rb +56 -0
data/lib/polipus/storage/mongo_store.rb +90 -0
data/lib/polipus/storage/rethink_store.rb +90 -0
data/lib/polipus/url_tracker.rb +21 -0
data/lib/polipus/url_tracker/bloomfilter.rb +27 -0
data/lib/polipus/url_tracker/redis_set.rb +27 -0
data/lib/polipus/version.rb +5 -0
data/polipus.gemspec +44 -0
data/spec/cassettes/11c3eb8bf35dfc179dc5ce44f6f5f458.yml +6144 -0
data/spec/cassettes/1f6e1d7743ecaa86594b4e68a6462689.yml +11320 -0
data/spec/cassettes/6adfecdb274dd26ffd3713169583ca91.yml +18236 -0
data/spec/cassettes/978ac0eeb5df63a019b754cc8a965b06.yml +18296 -0
data/spec/cassettes/b389efd1dcb8f09393b5aae1627c2a83.yml +36569 -0
data/spec/cassettes/bc6fb220895689be7eeb05b09969a18d.yml +61 -0
data/spec/cassettes/c5ce68499027d490adfbb6e5541881e4.yml +18165 -0
data/spec/cassettes/ce16b11a7df0b70fe90c7f90063fdb8c.yml +11758 -0
data/spec/cassettes/gzipped_on.yml +147 -0
data/spec/cassettes/http_cookies.yml +133 -0
data/spec/cassettes/http_tconnection_max_hits.yml +4138 -0
data/spec/cassettes/http_test.yml +1418 -0
data/spec/cassettes/http_test_redirect.yml +71 -0
data/spec/clear.rb +12 -0
data/spec/polipus/http_spec.rb +139 -0
data/spec/polipus/page_spec.rb +68 -0
data/spec/polipus/queue_overflow/manager_spec.rb +88 -0
data/spec/polipus/queue_overflow_spec.rb +66 -0
data/spec/polipus/robotex_spec.rb +85 -0
data/spec/polipus/signal_handler_spec.rb +15 -0
data/spec/polipus/storage/memory_store_spec.rb +87 -0
data/spec/polipus/storage/mongo_store_spec.rb +119 -0
data/spec/polipus/storage/rethink_store_spec.rb +117 -0
data/spec/polipus/url_tracker_spec.rb +29 -0
data/spec/polipus_spec.rb +107 -0
data/spec/spec_helper.rb +42 -0
metadata +348 -0

data/spec/polipus/robotex_spec.rb ADDED Viewed

@@ -0,0 +1,85 @@
+require 'spec_helper'
+require 'polipus/robotex'
+describe Polipus::Robotex do
+  let(:spec_domain) { 'http://www.example.com/' }
+  before(:each) do
+    robots = <<-END
+User-Agent: msnbot
+Crawl-Delay: 20
+User-Agent: bender
+Disallow: /my_shiny_metal_ass
+User-Agent: *
+Disallow: /login
+Allow: /
+Disallow: /locked
+Allow: /locked
+END
+    stub_request(:get, 'http://www.example.com/robots.txt')
+      .to_return(body: robots, status: [200, 'OK'], headers: { 'Content-Type' => 'text/plain' })
+  end
+  describe '#initialize' do
+    context 'when no arguments are supplied' do
+      it 'returns a Robotex with the default user-agent' do
+        expect(Polipus::Robotex.new.user_agent).to eq("Robotex/#{Polipus::Robotex::VERSION} (http://www.github.com/chriskite/robotex)")
+      end
+    end
+    context 'when a user-agent is specified' do
+      it 'returns a Robotex with the specified user-agent' do
+        ua = 'My User Agent'
+        expect(Polipus::Robotex.new(ua).user_agent).to eq(ua)
+      end
+    end
+  end
+  describe '#allowed?' do
+    context 'when the robots.txt disallows the user-agent to the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new('bender')
+        expect(robotex.allowed?(spec_domain + 'my_shiny_metal_ass')).to be_falsey
+      end
+    end
+    context 'when the robots.txt disallows the user-agent to some urls, but allows this one' do
+      it 'returns true' do
+        robotex = Polipus::Robotex.new('bender')
+        expect(robotex.allowed?(spec_domain + 'cigars')).to be_truthy
+      end
+    end
+    context 'when the robots.txt disallows any user-agent to the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new
+        expect(robotex.allowed?(spec_domain + 'login')).to be_falsey
+      end
+    end
+    context 'when the robots.txt disallows and then allows the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new
+        expect(robotex.allowed?(spec_domain + 'locked')).to be_falsey
+      end
+    end
+  end
+  describe '#delay' do
+    context 'when no Crawl-Delay is specified for the user-agent' do
+      it 'returns nil' do
+        robotex = Polipus::Robotex.new
+        expect(robotex.delay(spec_domain)).to be_nil
+      end
+      context 'when Crawl-Delay is specified for the user-agent' do
+        it 'returns the delay as a Fixnum' do
+          robotex = Polipus::Robotex.new('msnbot')
+          expect(robotex.delay(spec_domain)).to eq(20)
+        end
+      end
+    end
+  end
+end

data/spec/polipus/signal_handler_spec.rb ADDED Viewed

@@ -0,0 +1,15 @@
+require 'spec_helper'
+describe Polipus::SignalHandler do
+  context 'signal handler' do
+    it 'should be enabled by default' do
+      Polipus::PolipusCrawler.new('polipus-rspec', [])
+      expect(Polipus::SignalHandler.enabled?).to be true
+    end
+    it 'should be disabled if specified' do
+      Polipus::PolipusCrawler.new('polipus-rspec', [], enable_signal_handler: false)
+      expect(Polipus::SignalHandler.enabled?).to be false
+    end
+  end
+end

data/spec/polipus/storage/memory_store_spec.rb ADDED Viewed

@@ -0,0 +1,87 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'mongo'
+require 'polipus/storage/memory_store'
+describe Polipus::Storage::MemoryStore do
+  let(:storage) { Polipus::Storage.memory_store }
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
+    uuid = storage.add p
+    expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
+    expect(storage.count).to be 1
+    p = storage.get p
+    expect(p.url.to_s).to eq('http://www.google.com')
+    expect(p.body).to eq('<html></html>')
+  end
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    storage.add p
+    p = storage.get p
+    expect(p.code).to eq(301)
+  end
+  it 'should iterate over stored pages' do
+    storage.each do |k, page|
+      expect(k).to eq('ed646a3334ca891fd3467db131372140')
+      expect(page.url.to_s).to eq('http://www.google.com')
+    end
+  end
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    storage.remove p
+    expect(storage.get(p)).to be_nil
+    expect(storage.count).to be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
+    storage.include_query_string_in_uuid = false
+    storage.add p
+    expect(storage.exists?(p_no_query)).to be_truthy
+    storage.remove p
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
+    storage.include_query_string_in_uuid = false
+    storage.add p
+    expect(storage.exists?(p_no_query)).to be_truthy
+    storage.remove p
+  end
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
+    p.user_data.name = 'Test User Data'
+    storage.add p
+    expect(storage.exists?(p)).to be_truthy
+    p = storage.get(p)
+    expect(p.user_data.name).to eq('Test User Data')
+    storage.remove p
+  end
+  it 'should honor the except parameters' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages', ['body'])
+    p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
+    storage.add p
+    p = storage.get p
+    expect(p.body).to be_empty
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
+    expect(storage.exists?(p_other)).to be_falsey
+    storage.add p_other
+    expect(storage.exists?(p_other)).to be_truthy
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
+    expect(storage.exists?(p_other)).to be_truthy
+    storage.include_query_string_in_uuid = true
+    expect(storage.exists?(p_other)).to be_falsey
+  end
+end

data/spec/polipus/storage/mongo_store_spec.rb ADDED Viewed

@@ -0,0 +1,119 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'mongo'
+require 'polipus/storage/mongo_store'
+describe Polipus::Storage::MongoStore do
+  before(:all)do
+    @mongo = Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('_test_polipus')
+    @mongo['_test_pages'].drop
+    @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+  end
+  after(:all) do
+    @mongo['_test_pages'].drop
+  end
+  after(:each) do
+    @mongo['_test_pages'].drop
+  end
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
+    uuid = @storage.add p
+    expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
+    expect(@storage.count).to be 1
+    expect(@mongo['_test_pages'].count).to be 1
+    p = @storage.get p
+    expect(p.url.to_s).to eq('http://www.google.com')
+    expect(p.body).to eq('<html></html>')
+  end
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    @storage.add p
+    p = @storage.get p
+    expect(p.code).to eq(301)
+    expect(@mongo['_test_pages'].count).to be 1
+  end
+  it 'should iterate over stored pages' do
+    @storage.each do |k, page|
+      expect(k).to eq('ed646a3334ca891fd3467db131372140')
+      expect(page.url.to_s).to eq('http://www.google.com')
+    end
+  end
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    @storage.remove p
+    expect(@storage.get(p)).to be_nil
+    expect(@storage.count).to be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove p
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove p
+  end
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
+    p.user_data.name = 'Test User Data'
+    @storage.add p
+    expect(@storage.exists?(p)).to be_truthy
+    p = @storage.get(p)
+    expect(p.user_data.name).to eq('Test User Data')
+    @storage.remove p
+  end
+  it 'should honor the except parameters' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages', ['body'])
+    p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
+    storage.add p
+    p = storage.get p
+    expect(p.body).to be_empty
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
+    expect(@storage.exists?(p_other)).to be_falsey
+    @storage.add p_other
+    expect(@storage.exists?(p_other)).to be_truthy
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
+    expect(@storage.exists?(p_other)).to be_truthy
+    @storage.include_query_string_in_uuid = true
+    expect(@storage.exists?(p_other)).to be_falsey
+  end
+  it 'should set page.fetched_at based on the id creation' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+    p = page_factory 'http://www.user-doojo.com',  code: 200, body: '<html></html>'
+    storage.add p
+    expect(p.fetched_at).to be_nil
+    p = storage.get p
+    expect(p.fetched_at).not_to be_nil
+  end
+  it 'should NOT set page.fetched_at if already present' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+    p = page_factory 'http://www.user-doojooo.com',  code: 200, body: '<html></html>'
+    p.fetched_at = 10
+    storage.add p
+    p = storage.get p
+    expect(p.fetched_at).to be 10
+  end
+end

data/spec/polipus/storage/rethink_store_spec.rb ADDED Viewed

@@ -0,0 +1,117 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'polipus/storage/rethink_store'
+describe Polipus::Storage::RethinkStore do
+  before(:all)do
+    @r = RethinkDB::RQL.new
+    @rethink = @r.connect(host: 'localhost', port: 28_015, db: 'polipus_spec')
+    @r.db_create('polipus_spec').run(@rethink) unless @r.db_list.run(@rethink).include?('polipus_spec')
+    @table = 'test_pages'
+    @storage = Polipus::Storage.rethink_store(@rethink, @table)
+  end
+  after(:each) do
+    @r.table(@table).delete.run(@rethink)
+  end
+  it 'should store a page' do
+    page = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
+    uuid = @storage.add page
+    expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
+    expect(@storage.count).to eq(1)
+    expect(@r.table(@table).count.run(@rethink)).to eq(1)
+    page = @storage.get page
+    expect(page.url.to_s).to eq('http://www.google.com')
+    expect(page.body).to eq('<html></html>')
+  end
+  it 'should update a page' do
+    page = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    @storage.add page
+    page = @storage.get page
+    expect(page.code).to eq(301)
+    expect(@r.table(@table).count.run(@rethink)).to eq(1)
+  end
+  it 'should iterate over stored pages' do
+    @storage.each do |k, page|
+      expect(k).to eq('ed646a3334ca891fd3467db131372140')
+      expect(page.url.to_s).to eq('http://www.google.com')
+    end
+  end
+  it 'should delete a page' do
+    page = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    @storage.remove page
+    expect(@storage.get(page)).to be_nil
+    expect(@storage.count).to be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    page = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add page
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove page
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    page = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add page
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove page
+  end
+  it 'should store a page with user data associated' do
+    page = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
+    page.user_data.name = 'Test User Data'
+    @storage.add page
+    expect(@storage.exists?(page)).to be_truthy
+    page = @storage.get(page)
+    expect(page.user_data.name).to eq('Test User Data')
+    @storage.remove page
+  end
+  it 'should honor the except parameters' do
+    storage = Polipus::Storage.rethink_store(@rethink, @table, ['body'])
+    page = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
+    storage.add page
+    page = storage.get page
+    expect(page.body).to be_empty
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
+    expect(@storage.exists?(p_other)).to be_falsey
+    @storage.add p_other
+    expect(@storage.exists?(p_other)).to be_truthy
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
+    expect(@storage.exists?(p_other)).to be_truthy
+    @storage.include_query_string_in_uuid = true
+    expect(@storage.exists?(p_other)).to be_falsey
+  end
+  it 'should set page.fetched_at based on the id creation' do
+    storage = Polipus::Storage.rethink_store(@rethink, @table)
+    page = page_factory 'http://www.user-doojo.com',  code: 200, body: '<html></html>'
+    storage.add page
+    expect(page.fetched_at).to be_nil
+    page = storage.get page
+    expect(page.fetched_at).not_to be_nil
+  end
+  it 'should NOT set page.fetched_at if already present' do
+    storage = Polipus::Storage.rethink_store(@rethink, @table)
+    page = page_factory 'http://www.user-doojooo.com',  code: 200, body: '<html></html>'
+    page.fetched_at = 10
+    storage.add page
+    page = storage.get page
+    expect(page.fetched_at).to be 10
+  end
+end

data/spec/polipus/url_tracker_spec.rb ADDED Viewed

@@ -0,0 +1,29 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'polipus/url_tracker'
+describe Polipus::UrlTracker do
+  before(:all) do
+    @bf  = Polipus::UrlTracker.bloomfilter
+    @set = Polipus::UrlTracker.redis_set
+  end
+  after(:all) do
+    @bf.clear
+    @set.clear
+  end
+  it 'should work (bf)' do
+    url = 'http://www.asd.com/asd/lol'
+    @bf.visit url
+    expect(@bf.visited?(url)).to be_truthy
+    expect(@bf.visited?('http://www.google.com')).to be_falsey
+  end
+  it 'should work (redis_set)' do
+    url = 'http://www.asd.com/asd/lol'
+    @set.visit url
+    expect(@set.visited?(url)).to be_truthy
+    expect(@set.visited?('http://www.google.com')).to be_falsey
+  end
+end