RubyGems - parallel588_polipus - Versions diffs - 0.4.0 - Mend

parallel588_polipus 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (72) hide show

checksums.yaml +7 -0
data/.document +5 -0
data/.gitignore +53 -0
data/.rspec +2 -0
data/.rubocop.yml +17 -0
data/.rubocop_todo.yml +33 -0
data/.travis.yml +22 -0
data/AUTHORS.md +5 -0
data/CHANGELOG.md +61 -0
data/Gemfile +12 -0
data/LICENSE.txt +20 -0
data/README.md +70 -0
data/Rakefile +8 -0
data/examples/basic.rb +63 -0
data/examples/error_handling.rb +23 -0
data/examples/incremental.rb +63 -0
data/examples/robots_txt_handling.rb +14 -0
data/examples/survival.rb +10 -0
data/lib/polipus.rb +488 -0
data/lib/polipus/http.rb +282 -0
data/lib/polipus/page.rb +256 -0
data/lib/polipus/plugin.rb +14 -0
data/lib/polipus/plugins/cleaner.rb +25 -0
data/lib/polipus/plugins/sample.rb +15 -0
data/lib/polipus/plugins/sleeper.rb +22 -0
data/lib/polipus/queue_overflow.rb +26 -0
data/lib/polipus/queue_overflow/base.rb +7 -0
data/lib/polipus/queue_overflow/dev_null_queue.rb +34 -0
data/lib/polipus/queue_overflow/manager.rb +57 -0
data/lib/polipus/queue_overflow/mongo_queue.rb +60 -0
data/lib/polipus/queue_overflow/mongo_queue_capped.rb +29 -0
data/lib/polipus/queue_overflow/worker.rb +24 -0
data/lib/polipus/robotex.rb +145 -0
data/lib/polipus/signal_handler.rb +42 -0
data/lib/polipus/storage.rb +31 -0
data/lib/polipus/storage/base.rb +20 -0
data/lib/polipus/storage/dev_null.rb +35 -0
data/lib/polipus/storage/memory_store.rb +56 -0
data/lib/polipus/storage/mongo_store.rb +90 -0
data/lib/polipus/storage/rethink_store.rb +90 -0
data/lib/polipus/url_tracker.rb +21 -0
data/lib/polipus/url_tracker/bloomfilter.rb +27 -0
data/lib/polipus/url_tracker/redis_set.rb +27 -0
data/lib/polipus/version.rb +5 -0
data/polipus.gemspec +44 -0
data/spec/cassettes/11c3eb8bf35dfc179dc5ce44f6f5f458.yml +6144 -0
data/spec/cassettes/1f6e1d7743ecaa86594b4e68a6462689.yml +11320 -0
data/spec/cassettes/6adfecdb274dd26ffd3713169583ca91.yml +18236 -0
data/spec/cassettes/978ac0eeb5df63a019b754cc8a965b06.yml +18296 -0
data/spec/cassettes/b389efd1dcb8f09393b5aae1627c2a83.yml +36569 -0
data/spec/cassettes/bc6fb220895689be7eeb05b09969a18d.yml +61 -0
data/spec/cassettes/c5ce68499027d490adfbb6e5541881e4.yml +18165 -0
data/spec/cassettes/ce16b11a7df0b70fe90c7f90063fdb8c.yml +11758 -0
data/spec/cassettes/gzipped_on.yml +147 -0
data/spec/cassettes/http_cookies.yml +133 -0
data/spec/cassettes/http_tconnection_max_hits.yml +4138 -0
data/spec/cassettes/http_test.yml +1418 -0
data/spec/cassettes/http_test_redirect.yml +71 -0
data/spec/clear.rb +12 -0
data/spec/polipus/http_spec.rb +139 -0
data/spec/polipus/page_spec.rb +68 -0
data/spec/polipus/queue_overflow/manager_spec.rb +88 -0
data/spec/polipus/queue_overflow_spec.rb +66 -0
data/spec/polipus/robotex_spec.rb +85 -0
data/spec/polipus/signal_handler_spec.rb +15 -0
data/spec/polipus/storage/memory_store_spec.rb +87 -0
data/spec/polipus/storage/mongo_store_spec.rb +119 -0
data/spec/polipus/storage/rethink_store_spec.rb +117 -0
data/spec/polipus/url_tracker_spec.rb +29 -0
data/spec/polipus_spec.rb +107 -0
data/spec/spec_helper.rb +42 -0
metadata +348 -0

data/spec/polipus/robotex_spec.rb ADDED Viewed

@@ -0,0 +1,85 @@
+require 'spec_helper'
+require 'polipus/robotex'
+describe Polipus::Robotex do
+  let(:spec_domain) { 'http://www.example.com/' }
+  before(:each) do
+    robots = <<-END
+User-Agent: msnbot
+Crawl-Delay: 20
+User-Agent: bender
+Disallow: /my_shiny_metal_ass
+User-Agent: *
+Disallow: /login
+Allow: /
+Disallow: /locked
+Allow: /locked
+END
+    stub_request(:get, 'http://www.example.com/robots.txt')
+      .to_return(body: robots, status: [200, 'OK'], headers: { 'Content-Type' => 'text/plain' })
+  end
+  describe '#initialize' do
+    context 'when no arguments are supplied' do
+      it 'returns a Robotex with the default user-agent' do
+        expect(Polipus::Robotex.new.user_agent).to eq("Robotex/#{Polipus::Robotex::VERSION} (http://www.github.com/chriskite/robotex)")
+      end
+    end
+    context 'when a user-agent is specified' do
+      it 'returns a Robotex with the specified user-agent' do
+        ua = 'My User Agent'
+        expect(Polipus::Robotex.new(ua).user_agent).to eq(ua)
+      end
+    end
+  end
+  describe '#allowed?' do
+    context 'when the robots.txt disallows the user-agent to the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new('bender')
+        expect(robotex.allowed?(spec_domain + 'my_shiny_metal_ass')).to be_falsey
+      end
+    end
+    context 'when the robots.txt disallows the user-agent to some urls, but allows this one' do
+      it 'returns true' do
+        robotex = Polipus::Robotex.new('bender')
+        expect(robotex.allowed?(spec_domain + 'cigars')).to be_truthy
+      end
+    end
+    context 'when the robots.txt disallows any user-agent to the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new
+        expect(robotex.allowed?(spec_domain + 'login')).to be_falsey
+      end
+    end
+    context 'when the robots.txt disallows and then allows the url' do
+      it 'returns false' do
+        robotex = Polipus::Robotex.new
+        expect(robotex.allowed?(spec_domain + 'locked')).to be_falsey
+      end
+    end
+  end
+  describe '#delay' do
+    context 'when no Crawl-Delay is specified for the user-agent' do
+      it 'returns nil' do
+        robotex = Polipus::Robotex.new
+        expect(robotex.delay(spec_domain)).to be_nil
+      end
+      context 'when Crawl-Delay is specified for the user-agent' do
+        it 'returns the delay as a Fixnum' do
+          robotex = Polipus::Robotex.new('msnbot')
+          expect(robotex.delay(spec_domain)).to eq(20)
+        end
+      end
+    end
+  end
+end

data/spec/polipus/signal_handler_spec.rb ADDED Viewed

@@ -0,0 +1,15 @@
+require 'spec_helper'
+describe Polipus::SignalHandler do
+  context 'signal handler' do
+    it 'should be enabled by default' do
+      Polipus::PolipusCrawler.new('polipus-rspec', [])
+      expect(Polipus::SignalHandler.enabled?).to be true
+    end
+    it 'should be disabled if specified' do
+      Polipus::PolipusCrawler.new('polipus-rspec', [], enable_signal_handler: false)
+      expect(Polipus::SignalHandler.enabled?).to be false
+    end
+  end
+end

data/spec/polipus/storage/memory_store_spec.rb ADDED Viewed

@@ -0,0 +1,87 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'mongo'
+require 'polipus/storage/memory_store'
+describe Polipus::Storage::MemoryStore do
+  let(:storage) { Polipus::Storage.memory_store }
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
+    uuid = storage.add p
+    expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
+    expect(storage.count).to be 1
+    p = storage.get p
+    expect(p.url.to_s).to eq('http://www.google.com')
+    expect(p.body).to eq('<html></html>')
+  end
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    storage.add p
+    p = storage.get p
+    expect(p.code).to eq(301)
+  end
+  it 'should iterate over stored pages' do
+    storage.each do |k, page|
+      expect(k).to eq('ed646a3334ca891fd3467db131372140')
+      expect(page.url.to_s).to eq('http://www.google.com')
+    end
+  end
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    storage.remove p
+    expect(storage.get(p)).to be_nil
+    expect(storage.count).to be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
+    storage.include_query_string_in_uuid = false
+    storage.add p
+    expect(storage.exists?(p_no_query)).to be_truthy
+    storage.remove p
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
+    storage.include_query_string_in_uuid = false
+    storage.add p
+    expect(storage.exists?(p_no_query)).to be_truthy
+    storage.remove p
+  end
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
+    p.user_data.name = 'Test User Data'
+    storage.add p
+    expect(storage.exists?(p)).to be_truthy
+    p = storage.get(p)
+    expect(p.user_data.name).to eq('Test User Data')
+    storage.remove p
+  end
+  it 'should honor the except parameters' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages', ['body'])
+    p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
+    storage.add p
+    p = storage.get p
+    expect(p.body).to be_empty
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
+    expect(storage.exists?(p_other)).to be_falsey
+    storage.add p_other
+    expect(storage.exists?(p_other)).to be_truthy
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
+    expect(storage.exists?(p_other)).to be_truthy
+    storage.include_query_string_in_uuid = true
+    expect(storage.exists?(p_other)).to be_falsey
+  end
+end

data/spec/polipus/storage/mongo_store_spec.rb ADDED Viewed

@@ -0,0 +1,119 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'mongo'
+require 'polipus/storage/mongo_store'
+describe Polipus::Storage::MongoStore do
+  before(:all)do
+    @mongo = Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('_test_polipus')
+    @mongo['_test_pages'].drop
+    @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+  end
+  after(:all) do
+    @mongo['_test_pages'].drop
+  end
+  after(:each) do
+    @mongo['_test_pages'].drop
+  end
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
+    uuid = @storage.add p
+    expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
+    expect(@storage.count).to be 1
+    expect(@mongo['_test_pages'].count).to be 1
+    p = @storage.get p
+    expect(p.url.to_s).to eq('http://www.google.com')
+    expect(p.body).to eq('<html></html>')
+  end
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    @storage.add p
+    p = @storage.get p
+    expect(p.code).to eq(301)
+    expect(@mongo['_test_pages'].count).to be 1
+  end
+  it 'should iterate over stored pages' do
+    @storage.each do |k, page|
+      expect(k).to eq('ed646a3334ca891fd3467db131372140')
+      expect(page.url.to_s).to eq('http://www.google.com')
+    end
+  end
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    @storage.remove p
+    expect(@storage.get(p)).to be_nil
+    expect(@storage.count).to be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove p
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove p
+  end
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
+    p.user_data.name = 'Test User Data'
+    @storage.add p
+    expect(@storage.exists?(p)).to be_truthy
+    p = @storage.get(p)
+    expect(p.user_data.name).to eq('Test User Data')
+    @storage.remove p
+  end
+  it 'should honor the except parameters' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages', ['body'])
+    p = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
+    storage.add p
+    p = storage.get p
+    expect(p.body).to be_empty
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
+    expect(@storage.exists?(p_other)).to be_falsey
+    @storage.add p_other
+    expect(@storage.exists?(p_other)).to be_truthy
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
+    expect(@storage.exists?(p_other)).to be_truthy
+    @storage.include_query_string_in_uuid = true
+    expect(@storage.exists?(p_other)).to be_falsey
+  end
+  it 'should set page.fetched_at based on the id creation' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+    p = page_factory 'http://www.user-doojo.com',  code: 200, body: '<html></html>'
+    storage.add p
+    expect(p.fetched_at).to be_nil
+    p = storage.get p
+    expect(p.fetched_at).not_to be_nil
+  end
+  it 'should NOT set page.fetched_at if already present' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+    p = page_factory 'http://www.user-doojooo.com',  code: 200, body: '<html></html>'
+    p.fetched_at = 10
+    storage.add p
+    p = storage.get p
+    expect(p.fetched_at).to be 10
+  end
+end

data/spec/polipus/storage/rethink_store_spec.rb ADDED Viewed

@@ -0,0 +1,117 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'polipus/storage/rethink_store'
+describe Polipus::Storage::RethinkStore do
+  before(:all)do
+    @r = RethinkDB::RQL.new
+    @rethink = @r.connect(host: 'localhost', port: 28_015, db: 'polipus_spec')
+    @r.db_create('polipus_spec').run(@rethink) unless @r.db_list.run(@rethink).include?('polipus_spec')
+    @table = 'test_pages'
+    @storage = Polipus::Storage.rethink_store(@rethink, @table)
+  end
+  after(:each) do
+    @r.table(@table).delete.run(@rethink)
+  end
+  it 'should store a page' do
+    page = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
+    uuid = @storage.add page
+    expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
+    expect(@storage.count).to eq(1)
+    expect(@r.table(@table).count.run(@rethink)).to eq(1)
+    page = @storage.get page
+    expect(page.url.to_s).to eq('http://www.google.com')
+    expect(page.body).to eq('<html></html>')
+  end
+  it 'should update a page' do
+    page = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    @storage.add page
+    page = @storage.get page
+    expect(page.code).to eq(301)
+    expect(@r.table(@table).count.run(@rethink)).to eq(1)
+  end
+  it 'should iterate over stored pages' do
+    @storage.each do |k, page|
+      expect(k).to eq('ed646a3334ca891fd3467db131372140')
+      expect(page.url.to_s).to eq('http://www.google.com')
+    end
+  end
+  it 'should delete a page' do
+    page = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
+    @storage.remove page
+    expect(@storage.get(page)).to be_nil
+    expect(@storage.count).to be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    page = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add page
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove page
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    page = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add page
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove page
+  end
+  it 'should store a page with user data associated' do
+    page = page_factory 'http://www.user.com',  code: 200, body: '<html></html>'
+    page.user_data.name = 'Test User Data'
+    @storage.add page
+    expect(@storage.exists?(page)).to be_truthy
+    page = @storage.get(page)
+    expect(page.user_data.name).to eq('Test User Data')
+    @storage.remove page
+  end
+  it 'should honor the except parameters' do
+    storage = Polipus::Storage.rethink_store(@rethink, @table, ['body'])
+    page = page_factory 'http://www.user-doo.com',  code: 200, body: '<html></html>'
+    storage.add page
+    page = storage.get page
+    expect(page.body).to be_empty
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
+    expect(@storage.exists?(p_other)).to be_falsey
+    @storage.add p_other
+    expect(@storage.exists?(p_other)).to be_truthy
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
+    expect(@storage.exists?(p_other)).to be_truthy
+    @storage.include_query_string_in_uuid = true
+    expect(@storage.exists?(p_other)).to be_falsey
+  end
+  it 'should set page.fetched_at based on the id creation' do
+    storage = Polipus::Storage.rethink_store(@rethink, @table)
+    page = page_factory 'http://www.user-doojo.com',  code: 200, body: '<html></html>'
+    storage.add page
+    expect(page.fetched_at).to be_nil
+    page = storage.get page
+    expect(page.fetched_at).not_to be_nil
+  end
+  it 'should NOT set page.fetched_at if already present' do
+    storage = Polipus::Storage.rethink_store(@rethink, @table)
+    page = page_factory 'http://www.user-doojooo.com',  code: 200, body: '<html></html>'
+    page.fetched_at = 10
+    storage.add page
+    page = storage.get page
+    expect(page.fetched_at).to be 10
+  end
+end

data/spec/polipus/url_tracker_spec.rb ADDED Viewed

@@ -0,0 +1,29 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'polipus/url_tracker'
+describe Polipus::UrlTracker do
+  before(:all) do
+    @bf  = Polipus::UrlTracker.bloomfilter
+    @set = Polipus::UrlTracker.redis_set
+  end
+  after(:all) do
+    @bf.clear
+    @set.clear
+  end
+  it 'should work (bf)' do
+    url = 'http://www.asd.com/asd/lol'
+    @bf.visit url
+    expect(@bf.visited?(url)).to be_truthy
+    expect(@bf.visited?('http://www.google.com')).to be_falsey
+  end
+  it 'should work (redis_set)' do
+    url = 'http://www.asd.com/asd/lol'
+    @set.visit url
+    expect(@set.visited?(url)).to be_truthy
+    expect(@set.visited?('http://www.google.com')).to be_falsey
+  end
+end