RubyGems - polipus - Versions diffs - 0.0.1 - Mend

polipus 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (55) hide show

checksums.yaml +15 -0
data/.document +5 -0
data/.gitignore +53 -0
data/.rspec +2 -0
data/Gemfile +3 -0
data/LICENSE.txt +20 -0
data/README.md +55 -0
data/README.rdoc +3 -0
data/Rakefile +9 -0
data/examples/basic.rb +58 -0
data/examples/survival.rb +9 -0
data/lib/polipus.rb +451 -0
data/lib/polipus/http.rb +195 -0
data/lib/polipus/page.rb +219 -0
data/lib/polipus/plugin.rb +13 -0
data/lib/polipus/plugins/cleaner.rb +25 -0
data/lib/polipus/plugins/sample.rb +17 -0
data/lib/polipus/plugins/sleeper.rb +22 -0
data/lib/polipus/queue_overflow.rb +24 -0
data/lib/polipus/queue_overflow/base.rb +6 -0
data/lib/polipus/queue_overflow/dev_null_queue.rb +33 -0
data/lib/polipus/queue_overflow/manager.rb +50 -0
data/lib/polipus/queue_overflow/mongo_queue.rb +61 -0
data/lib/polipus/queue_overflow/mongo_queue_capped.rb +28 -0
data/lib/polipus/storage.rb +31 -0
data/lib/polipus/storage/base.rb +17 -0
data/lib/polipus/storage/dev_null.rb +35 -0
data/lib/polipus/storage/mongo_store.rb +86 -0
data/lib/polipus/storage/s3_store.rb +100 -0
data/lib/polipus/url_tracker.rb +20 -0
data/lib/polipus/url_tracker/bloomfilter.rb +27 -0
data/lib/polipus/url_tracker/redis_set.rb +27 -0
data/lib/polipus/version.rb +4 -0
data/polipus.gemspec +39 -0
data/spec/cassettes/08b228db424a926e1ed6ab63b38d847e.yml +166 -0
data/spec/cassettes/20aa41f181b49f00078c3ca30bad5afe.yml +166 -0
data/spec/cassettes/4640919145753505af2d0f8423de37f3.yml +270 -0
data/spec/cassettes/66aae15a03f4aab8efd15e40d2d7882a.yml +194 -0
data/spec/cassettes/76b7c197c95a5bf9b1e882c567192d72.yml +183 -0
data/spec/cassettes/9b1d523b7f5db7214f8a8bd9272cccba.yml +221 -0
data/spec/cassettes/ab333f89535a2efb284913fede6aa7c7.yml +221 -0
data/spec/cassettes/ae5d7cffde3f53122cdf79f3d1367e8e.yml +221 -0
data/spec/cassettes/ffe3d588b6df4b9de35e5a7ccaf5a81b.yml +695 -0
data/spec/cassettes/http_test.yml +1418 -0
data/spec/cassettes/http_test_redirect.yml +71 -0
data/spec/clear.rb +11 -0
data/spec/http_spec.rb +31 -0
data/spec/page_spec.rb +22 -0
data/spec/queue_overflow_manager_spec.rb +89 -0
data/spec/queue_overflow_spec.rb +71 -0
data/spec/spec_helper.rb +34 -0
data/spec/storage_mongo_spec.rb +102 -0
data/spec/storage_s3_spec.rb +115 -0
data/spec/url_tracker_spec.rb +28 -0
metadata +313 -0

data/spec/cassettes/http_test_redirect.yml ADDED Viewed

@@ -0,0 +1,71 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 300
+      message: Multiple Choices
+    headers:
+      Date:
+      - Mon, 10 Jun 2013 08:58:25 GMT
+      Server:
+      - Apache/2.2.22 (Ubuntu)
+      Cache-Control:
+      - no-cache
+      Location:
+      - http://greenbytes.de/tech/tc/httpredirects/300.txt
+      Content-Length:
+      - '27'
+    body:
+      encoding: US-ASCII
+      string: ! '300 Redirect Response Body
+'
+    http_version:
+  recorded_at: Mon, 10 Jun 2013 08:58:25 GMT
+- request:
+    method: get
+    uri: http://greenbytes.de/tech/tc/httpredirects/300.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 10 Jun 2013 08:58:25 GMT
+      Server:
+      - Apache/2.2.22 (Ubuntu)
+      Last-Modified:
+      - Tue, 08 Jan 2013 17:31:05 GMT
+      Etag:
+      - ! '"b8306c-31-4d2ca4f7df2ca"'
+      Accept-Ranges:
+      - bytes
+      Content-Length:
+      - '49'
+      Content-Type:
+      - text/plain
+    body:
+      encoding: US-ASCII
+      string: ! "You have reached the target\r\nof a 300 redirect.\r\n"
+    http_version:
+  recorded_at: Mon, 10 Jun 2013 08:58:25 GMT
+recorded_with: VCR 2.5.0

data/spec/clear.rb ADDED Viewed

@@ -0,0 +1,11 @@
+require "yaml"
+Dir.glob('./cassettes/*.yml').each do|f|
+  next unless f =~ /[a-f0-9]{32}/
+  d = YAML.load_file(f)
+  d['http_interactions'].each do |r|
+    r['request'].delete('headers')
+    r['response'].delete('headers')
+  end
+  File.open(f, 'w') {|fw| fw.write(d.to_yaml) }
+  #puts d.to_yaml
+end

data/spec/http_spec.rb ADDED Viewed

@@ -0,0 +1,31 @@
+require "spec_helper"
+require "mongo"
+require "polipus/http"
+require "polipus/page"
+describe Polipus::HTTP do
+  it 'should download a page' do
+    VCR.use_cassette('http_test') do
+      http = Polipus::HTTP.new
+      page = http.fetch_page("http://sfbay.craigslist.org/apa/")
+      page.should be_an_instance_of(Polipus::Page)
+      page.doc.search("title").text.strip.should be == "SF bay area apts/housing for rent classifieds  - craigslist"
+    end
+  end
+  it 'should follow a redirect' do
+    VCR.use_cassette('http_test_redirect') do
+      http = Polipus::HTTP.new
+      page = http.fetch_page("http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis")
+      page.should be_an_instance_of(Polipus::Page)
+      page.code.should be == 200
+      page.url.to_s.should be == "http://greenbytes.de/tech/tc/httpredirects/300.txt"
+      page.body.strip.should be == "You have reached the target\r\nof a 300 redirect."
+    end
+  end
+end

data/spec/page_spec.rb ADDED Viewed

@@ -0,0 +1,22 @@
+require "spec_helper"
+require "polipus/page"
+describe Polipus::Page do
+  it 'should honor domain_aliases attribute' do
+body = <<EOF
+<html>
+  <body>
+    <a href="/page/1">1</a>
+    <a href="/page/2">2</a>
+    <a href="http://www.google.com/page/3">3</a>
+    <a href="http://google.com/page/3">4</a>
+    <a href="http://not.google.com/page/3">4</a>
+  </body>
+</html>
+EOF
+    h = {'content-type' => ['text/html']}
+    domain_aliases = %w(www.google.com google.com)
+    p = Polipus::Page.new 'http://www.google.com/', :code => 200, :body => body, :headers => h, :domain_aliases => domain_aliases
+    p.links.count.should be == 4
+  end
+end

data/spec/queue_overflow_manager_spec.rb ADDED Viewed

@@ -0,0 +1,89 @@
+require "spec_helper"
+require "polipus/queue_overflow"
+require "redis-queue"
+describe Polipus::QueueOverflow::Manager do
+  before(:all) do
+    @mongo = Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('_test_polipus')
+    @mongo['_test_pages'].drop
+    @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+    @redis_q = Redis::Queue.new("queue_test","bp_queue_test", :redis => Redis.new())
+    @queue_overflow   = Polipus::QueueOverflow.mongo_queue(nil, "queue_test")
+    @polipus = flexmock("polipus")
+    @polipus.should_receive(:queue_overflow_adapter).and_return(@queue_overflow)
+    @polipus.should_receive(:storage).and_return(@storage)
+    @manager = Polipus::QueueOverflow::Manager.new(@polipus, @redis_q, 10)
+  end
+  before(:each) do
+    @queue_overflow.clear
+    @redis_q.clear
+    @storage.clear
+  end
+  after(:all) do
+    @queue_overflow.clear
+    @redis_q.clear
+  end
+  it 'should remove 10 items' do
+    @manager.perform.should be == [0,0]
+    20.times {|i| @redis_q << page_factory("http://www.user-doo.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json  }
+    @manager.perform.should be == [10, 0]
+    @queue_overflow.size.should be == 10
+    @redis_q.size.should be == 10
+  end
+  it 'should restore 10 items' do
+    @manager.perform.should be == [0,0]
+    10.times {|i| @queue_overflow << page_factory("http://www.user-doo-bla.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json }
+    @manager.perform.should be == [0, 10]
+    @queue_overflow.size.should be == 0
+    @redis_q.size.should be == 10
+    @manager.perform.should be == [0, 0]
+  end
+  it 'should restore 3 items' do
+    @manager.perform.should be == [0,0]
+    3.times {|i| @queue_overflow << page_factory("http://www.user-doo-bu.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json }
+    @manager.perform.should be == [0, 3]
+    @queue_overflow.size.should be == 0
+    @redis_q.size.should be == 3
+    @manager.perform.should be == [0, 0]
+  end
+  it 'should restore 0 items' do
+    @manager.perform.should be == [0,0]
+    10.times {|i|
+      p = page_factory("http://www.user-doo-bu.com/page_#{i}",  :code => 200, :body => '<html></html>')
+      @storage.add p
+      @queue_overflow << p.to_json
+    }
+    @manager.perform.should be == [0, 0]
+    @queue_overflow.size.should be == 0
+    @redis_q.size.should be == 0
+    @manager.perform.should be == [0, 0]
+  end
+  it 'should filter an url based on the spec' do
+    @queue_overflow.clear
+    @redis_q.clear
+    10.times {|i| @queue_overflow << page_factory("http://www.user-doo.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json  }
+    @manager.url_filter do |page|
+      page.url.to_s.end_with?("page_0") ? false : true
+    end
+    @manager.perform.should be == [0,9]
+    @queue_overflow.size.should be == 0
+    @redis_q.size.should be == 9
+    @manager.url_filter do |page|
+      true
+    end
+  end
+end

data/spec/queue_overflow_spec.rb ADDED Viewed

@@ -0,0 +1,71 @@
+require "spec_helper"
+require "polipus/queue_overflow"
+describe Polipus::QueueOverflow do
+  before(:all) do
+    @queue_overflow        = Polipus::QueueOverflow.mongo_queue(nil, "queue_test")
+    @queue_overflow_capped = Polipus::QueueOverflow.mongo_queue_capped(nil, "queue_test_c", {:max => 20})
+    @queue_overflow_uniq   = Polipus::QueueOverflow.mongo_queue(nil, "queue_test_u", {:ensure_uniq => true })
+  end
+  before(:each) do
+    @queue_overflow.clear
+    @queue_overflow_capped.clear
+    @queue_overflow_uniq.clear
+  end
+  after(:all) do
+    @queue_overflow.clear
+    @queue_overflow_uniq.clear
+    @queue_overflow_capped.clear
+  end
+  it 'should work' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      q.empty?.should be_true
+      q.pop.should be_nil
+      q << "test"
+      q.size.should be == 1
+      q.pop.should be == "test"
+      q.empty?.should be_true
+      q.pop.should be_nil
+      q.size.should be == 0
+      q.empty?.should be_true
+    end
+  end
+  it 'should act as a queue' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      10.times { |i| q << "message_#{i}" }
+      q.size.should be == 10
+      q.pop.should be == "message_0"
+    end
+  end
+  it 'should work with complex paylod' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      a = {'a' => [1,2,3], 'b' => 'a_string'}
+      q << a.to_json
+      b = q.pop
+      JSON.parse(b).should be == a
+    end
+  end
+  it 'should honor max items if it is capped' do
+    30.times { |i| @queue_overflow_capped << "message_#{i}" }
+    @queue_overflow_capped.size.should be == 20
+    @queue_overflow_capped.pop.should be == "message_10"
+  end
+  it 'should contains only unique items' do
+    20.times {@queue_overflow_uniq << "A"}
+    20.times {@queue_overflow_uniq << "B"}
+    @queue_overflow_uniq.size.should be == 2
+  end
+end

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,34 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require "digest/md5"
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+  config.mock_with :flexmock
+  config.around(:each) do |example|
+    VCR.use_cassette(Digest::MD5.hexdigest(example.metadata[:full_description])) do
+      example.run
+    end
+  end
+end
+require "vcr"
+require "polipus"
+VCR.configure do |c|
+  c.cassette_library_dir = "#{File.dirname(__FILE__)}/cassettes"
+  c.hook_into :webmock
+end
+def page_factory url, params = {}
+  Polipus::Page.new url, params
+end

data/spec/storage_mongo_spec.rb ADDED Viewed

@@ -0,0 +1,102 @@
+require "spec_helper"
+require "mongo"
+require "polipus/storage/mongo_store"
+describe Polipus::Storage::MongoStore do
+  before(:all)do
+    @mongo = Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('_test_polipus')
+    @mongo['_test_pages'].drop
+    @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+  end
+  after(:all) do
+    @mongo['_test_pages'].drop
+  end
+  after(:each) do
+    @mongo['_test_pages'].drop
+  end
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com', :code => 200, :body => '<html></html>'
+    uuid = @storage.add p
+    uuid.should be == 'ed646a3334ca891fd3467db131372140'
+    @storage.count.should be 1
+    @mongo['_test_pages'].count.should be 1
+    p = @storage.get p
+    p.url.to_s.should be == 'http://www.google.com'
+    p.body.should be == '<html></html>'
+  end
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    @storage.add p
+    p = @storage.get p
+    p.code.should be == 301
+    @mongo['_test_pages'].count.should be 1
+  end
+  it 'should iterate over stored pages' do
+    @storage.each do |k, page|
+      k.should be == "ed646a3334ca891fd3467db131372140"
+      page.url.to_s.should be == 'http://www.google.com'
+    end
+  end
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    @storage.remove p
+    @storage.get(p).should be_nil
+    @storage.count.should be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol', :code => 200, :body => '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', :code => 200, :body => '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    @storage.exists?(p_no_query).should be_true
+    @storage.remove p
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol', :code => 200, :body => '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', :code => 200, :body => '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    @storage.exists?(p_no_query).should be_true
+    @storage.remove p
+  end
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com',  :code => 200, :body => '<html></html>'
+    p.user_data.name = 'Test User Data'
+    @storage.add p
+    @storage.exists?(p).should be_true
+    p = @storage.get(p)
+    p.user_data.name.should be == 'Test User Data'
+    @storage.remove p
+  end
+  it 'should honor the except parameters' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages', ['body'])
+    p = page_factory 'http://www.user-doo.com',  :code => 200, :body => '<html></html>'
+    storage.add p
+    p = storage.get p
+    p.body.should be_empty
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', :code => 200, :body => '<html></html>'
+    @storage.exists?(p_other).should be_false
+    @storage.add p_other
+    @storage.exists?(p_other).should be_true
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', :code => 200, :body => '<html></html>'
+    @storage.exists?(p_other).should be_true
+    @storage.include_query_string_in_uuid = true
+    @storage.exists?(p_other).should be_false
+  end
+end

data/spec/storage_s3_spec.rb ADDED Viewed

@@ -0,0 +1,115 @@
+require "spec_helper"
+require "aws/s3"
+require "polipus/storage/s3_store"
+describe Polipus::Storage::S3Store do
+  before(:each) do
+    @storage = Polipus::Storage.s3_store(
+      '_test_pages',
+      {
+        :access_key_id => 'XXXXXXX',
+        :secret_access_key => 'XXXX'
+      }
+    )
+  end
+  after(:each) {@storage.clear}
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com', :code => 200, :body => '<html></html>'
+    uuid = @storage.add p
+    uuid.should be == 'ed646a3334ca891fd3467db131372140'
+    @storage.count.should be 1
+    p = @storage.get p
+    p.url.to_s.should be == 'http://www.google.com'
+    p.body.should be == '<html></html>'
+    @storage.remove p
+  end
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    @storage.add p
+    p = @storage.get p
+    p.code.should be == 301
+    @storage.count.should be == 1
+    @storage.remove p
+  end
+  it 'should iterate over stored pages' do
+    10.times {|i| @storage.add page_factory("http://www.google.com/p_#{i}", :code => 200, :body => "<html>#{i}</html>")}
+    @storage.count.should be 10
+    @storage.each do |k, page|
+      k.should be =~ /[a-f0-9]{32}/
+    end
+  end
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    @storage.add p
+    @storage.remove p
+    @storage.get(p).should be_nil
+    @storage.count.should be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol', :code => 200, :body => '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', :code => 200, :body => '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    @storage.exists?(p_no_query).should be_true
+    @storage.remove p
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol', :code => 200, :body => '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', :code => 200, :body => '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    @storage.exists?(p_no_query).should be_true
+    @storage.remove p
+  end
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com',  :code => 200, :body => '<html></html>'
+    p.user_data.name = 'Test User Data'
+    @storage.add p
+    @storage.exists?(p).should be_true
+    p = @storage.get(p)
+    p.user_data.name.should be == 'Test User Data'
+    @storage.remove p
+  end
+  it 'should honor the except parameters' do
+    storage = storage = Polipus::Storage.s3_store(
+        '_test_pages',
+         {
+          :access_key_id => 'XXXXXXX',
+          :secret_access_key => 'XXXX'
+        },
+        ['body']
+      )
+    p = page_factory 'http://www.user-doo.com',  :code => 200, :body => '<html></html>'
+    storage.add p
+    p = storage.get p
+    p.body.should be_nil
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', :code => 200, :body => '<html></html>'
+    @storage.exists?(p_other).should be_false
+    @storage.add p_other
+    @storage.exists?(p_other).should be_true
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', :code => 200, :body => '<html></html>'
+    @storage.exists?(p_other).should be_true
+    @storage.include_query_string_in_uuid = true
+    @storage.exists?(p_other).should be_false
+    @storage.remove p_other
+  end
+end