RubyGems - polipus - Versions diffs - 0.0.1 - Mend

polipus 0.0.1

Files changed (55) hide show

checksums.yaml +15 -0
data/.document +5 -0
data/.gitignore +53 -0
data/.rspec +2 -0
data/Gemfile +3 -0
data/LICENSE.txt +20 -0
data/README.md +55 -0
data/README.rdoc +3 -0
data/Rakefile +9 -0
data/examples/basic.rb +58 -0
data/examples/survival.rb +9 -0
data/lib/polipus.rb +451 -0
data/lib/polipus/http.rb +195 -0
data/lib/polipus/page.rb +219 -0
data/lib/polipus/plugin.rb +13 -0
data/lib/polipus/plugins/cleaner.rb +25 -0
data/lib/polipus/plugins/sample.rb +17 -0
data/lib/polipus/plugins/sleeper.rb +22 -0
data/lib/polipus/queue_overflow.rb +24 -0
data/lib/polipus/queue_overflow/base.rb +6 -0
data/lib/polipus/queue_overflow/dev_null_queue.rb +33 -0
data/lib/polipus/queue_overflow/manager.rb +50 -0
data/lib/polipus/queue_overflow/mongo_queue.rb +61 -0
data/lib/polipus/queue_overflow/mongo_queue_capped.rb +28 -0
data/lib/polipus/storage.rb +31 -0
data/lib/polipus/storage/base.rb +17 -0
data/lib/polipus/storage/dev_null.rb +35 -0
data/lib/polipus/storage/mongo_store.rb +86 -0
data/lib/polipus/storage/s3_store.rb +100 -0
data/lib/polipus/url_tracker.rb +20 -0
data/lib/polipus/url_tracker/bloomfilter.rb +27 -0
data/lib/polipus/url_tracker/redis_set.rb +27 -0
data/lib/polipus/version.rb +4 -0
data/polipus.gemspec +39 -0
data/spec/cassettes/08b228db424a926e1ed6ab63b38d847e.yml +166 -0
data/spec/cassettes/20aa41f181b49f00078c3ca30bad5afe.yml +166 -0
data/spec/cassettes/4640919145753505af2d0f8423de37f3.yml +270 -0
data/spec/cassettes/66aae15a03f4aab8efd15e40d2d7882a.yml +194 -0
data/spec/cassettes/76b7c197c95a5bf9b1e882c567192d72.yml +183 -0
data/spec/cassettes/9b1d523b7f5db7214f8a8bd9272cccba.yml +221 -0
data/spec/cassettes/ab333f89535a2efb284913fede6aa7c7.yml +221 -0
data/spec/cassettes/ae5d7cffde3f53122cdf79f3d1367e8e.yml +221 -0
data/spec/cassettes/ffe3d588b6df4b9de35e5a7ccaf5a81b.yml +695 -0
data/spec/cassettes/http_test.yml +1418 -0
data/spec/cassettes/http_test_redirect.yml +71 -0
data/spec/clear.rb +11 -0
data/spec/http_spec.rb +31 -0
data/spec/page_spec.rb +22 -0
data/spec/queue_overflow_manager_spec.rb +89 -0
data/spec/queue_overflow_spec.rb +71 -0
data/spec/spec_helper.rb +34 -0
data/spec/storage_mongo_spec.rb +102 -0
data/spec/storage_s3_spec.rb +115 -0
data/spec/url_tracker_spec.rb +28 -0
metadata +313 -0

data/spec/cassettes/http_test_redirect.yml ADDED Viewed

@@ -0,0 +1,71 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 300
+      message: Multiple Choices
+    headers:
+      Date:
+      - Mon, 10 Jun 2013 08:58:25 GMT
+      Server:
+      - Apache/2.2.22 (Ubuntu)
+      Cache-Control:
+      - no-cache
+      Location:
+      - http://greenbytes.de/tech/tc/httpredirects/300.txt
+      Content-Length:
+      - '27'
+    body:
+      encoding: US-ASCII
+      string: ! '300 Redirect Response Body
+'
+    http_version:
+  recorded_at: Mon, 10 Jun 2013 08:58:25 GMT
+- request:
+    method: get
+    uri: http://greenbytes.de/tech/tc/httpredirects/300.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 10 Jun 2013 08:58:25 GMT
+      Server:
+      - Apache/2.2.22 (Ubuntu)
+      Last-Modified:
+      - Tue, 08 Jan 2013 17:31:05 GMT
+      Etag:
+      - ! '"b8306c-31-4d2ca4f7df2ca"'
+      Accept-Ranges:
+      - bytes
+      Content-Length:
+      - '49'
+      Content-Type:
+      - text/plain
+    body:
+      encoding: US-ASCII
+      string: ! "You have reached the target\r\nof a 300 redirect.\r\n"
+    http_version:
+  recorded_at: Mon, 10 Jun 2013 08:58:25 GMT
+recorded_with: VCR 2.5.0

data/spec/clear.rb ADDED Viewed

@@ -0,0 +1,11 @@
+require "yaml"
+Dir.glob('./cassettes/*.yml').each do|f|
+  next unless f =~ /[a-f0-9]{32}/
+  d = YAML.load_file(f)
+  d['http_interactions'].each do |r|
+    r['request'].delete('headers')
+    r['response'].delete('headers')
+  end
+  File.open(f, 'w') {|fw| fw.write(d.to_yaml) }
+  #puts d.to_yaml
+end

data/spec/http_spec.rb ADDED Viewed

@@ -0,0 +1,31 @@
+require "spec_helper"
+require "mongo"
+require "polipus/http"
+require "polipus/page"
+describe Polipus::HTTP do
+  it 'should download a page' do
+    VCR.use_cassette('http_test') do
+      http = Polipus::HTTP.new
+      page = http.fetch_page("http://sfbay.craigslist.org/apa/")
+      page.should be_an_instance_of(Polipus::Page)
+      page.doc.search("title").text.strip.should be == "SF bay area apts/housing for rent classifieds  - craigslist"
+    end
+  end
+  it 'should follow a redirect' do
+    VCR.use_cassette('http_test_redirect') do
+      http = Polipus::HTTP.new
+      page = http.fetch_page("http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis")
+      page.should be_an_instance_of(Polipus::Page)
+      page.code.should be == 200
+      page.url.to_s.should be == "http://greenbytes.de/tech/tc/httpredirects/300.txt"
+      page.body.strip.should be == "You have reached the target\r\nof a 300 redirect."
+    end
+  end
+end

data/spec/page_spec.rb ADDED Viewed

@@ -0,0 +1,22 @@
+require "spec_helper"
+require "polipus/page"
+describe Polipus::Page do
+  it 'should honor domain_aliases attribute' do
+body = <<EOF
+<html>
+  <body>
+    <a href="/page/1">1</a>
+    <a href="/page/2">2</a>
+    <a href="http://www.google.com/page/3">3</a>
+    <a href="http://google.com/page/3">4</a>
+    <a href="http://not.google.com/page/3">4</a>
+  </body>
+</html>
+EOF
+    h = {'content-type' => ['text/html']}
+    domain_aliases = %w(www.google.com google.com)
+    p = Polipus::Page.new 'http://www.google.com/', :code => 200, :body => body, :headers => h, :domain_aliases => domain_aliases
+    p.links.count.should be == 4
+  end
+end

data/spec/queue_overflow_manager_spec.rb ADDED Viewed

@@ -0,0 +1,89 @@
+require "spec_helper"
+require "polipus/queue_overflow"
+require "redis-queue"
+describe Polipus::QueueOverflow::Manager do
+  before(:all) do
+    @mongo = Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('_test_polipus')
+    @mongo['_test_pages'].drop
+    @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+    @redis_q = Redis::Queue.new("queue_test","bp_queue_test", :redis => Redis.new())
+    @queue_overflow   = Polipus::QueueOverflow.mongo_queue(nil, "queue_test")
+    @polipus = flexmock("polipus")
+    @polipus.should_receive(:queue_overflow_adapter).and_return(@queue_overflow)
+    @polipus.should_receive(:storage).and_return(@storage)
+    @manager = Polipus::QueueOverflow::Manager.new(@polipus, @redis_q, 10)
+  end
+  before(:each) do
+    @queue_overflow.clear
+    @redis_q.clear
+    @storage.clear
+  end
+  after(:all) do
+    @queue_overflow.clear
+    @redis_q.clear
+  end
+  it 'should remove 10 items' do
+    @manager.perform.should be == [0,0]
+    20.times {|i| @redis_q << page_factory("http://www.user-doo.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json  }
+    @manager.perform.should be == [10, 0]
+    @queue_overflow.size.should be == 10
+    @redis_q.size.should be == 10
+  end
+  it 'should restore 10 items' do
+    @manager.perform.should be == [0,0]
+    10.times {|i| @queue_overflow << page_factory("http://www.user-doo-bla.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json }
+    @manager.perform.should be == [0, 10]
+    @queue_overflow.size.should be == 0
+    @redis_q.size.should be == 10
+    @manager.perform.should be == [0, 0]
+  end
+  it 'should restore 3 items' do
+    @manager.perform.should be == [0,0]
+    3.times {|i| @queue_overflow << page_factory("http://www.user-doo-bu.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json }
+    @manager.perform.should be == [0, 3]
+    @queue_overflow.size.should be == 0
+    @redis_q.size.should be == 3
+    @manager.perform.should be == [0, 0]
+  end
+  it 'should restore 0 items' do
+    @manager.perform.should be == [0,0]
+    10.times {|i|
+      p = page_factory("http://www.user-doo-bu.com/page_#{i}",  :code => 200, :body => '<html></html>')
+      @storage.add p
+      @queue_overflow << p.to_json
+    }
+    @manager.perform.should be == [0, 0]
+    @queue_overflow.size.should be == 0
+    @redis_q.size.should be == 0
+    @manager.perform.should be == [0, 0]
+  end
+  it 'should filter an url based on the spec' do
+    @queue_overflow.clear
+    @redis_q.clear
+    10.times {|i| @queue_overflow << page_factory("http://www.user-doo.com/page_#{i}",  :code => 200, :body => '<html></html>').to_json  }
+    @manager.url_filter do |page|
+      page.url.to_s.end_with?("page_0") ? false : true
+    end
+    @manager.perform.should be == [0,9]
+    @queue_overflow.size.should be == 0
+    @redis_q.size.should be == 9
+    @manager.url_filter do |page|
+      true
+    end
+  end
+end

data/spec/queue_overflow_spec.rb ADDED Viewed

@@ -0,0 +1,71 @@
+require "spec_helper"
+require "polipus/queue_overflow"
+describe Polipus::QueueOverflow do
+  before(:all) do
+    @queue_overflow        = Polipus::QueueOverflow.mongo_queue(nil, "queue_test")
+    @queue_overflow_capped = Polipus::QueueOverflow.mongo_queue_capped(nil, "queue_test_c", {:max => 20})
+    @queue_overflow_uniq   = Polipus::QueueOverflow.mongo_queue(nil, "queue_test_u", {:ensure_uniq => true })
+  end
+  before(:each) do
+    @queue_overflow.clear
+    @queue_overflow_capped.clear
+    @queue_overflow_uniq.clear
+  end
+  after(:all) do
+    @queue_overflow.clear
+    @queue_overflow_uniq.clear
+    @queue_overflow_capped.clear
+  end
+  it 'should work' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      q.empty?.should be_true
+      q.pop.should be_nil
+      q << "test"
+      q.size.should be == 1
+      q.pop.should be == "test"
+      q.empty?.should be_true
+      q.pop.should be_nil
+      q.size.should be == 0
+      q.empty?.should be_true
+    end
+  end
+  it 'should act as a queue' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      10.times { |i| q << "message_#{i}" }
+      q.size.should be == 10
+      q.pop.should be == "message_0"
+    end
+  end
+  it 'should work with complex paylod' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      a = {'a' => [1,2,3], 'b' => 'a_string'}
+      q << a.to_json
+      b = q.pop
+      JSON.parse(b).should be == a
+    end
+  end
+  it 'should honor max items if it is capped' do
+    30.times { |i| @queue_overflow_capped << "message_#{i}" }
+    @queue_overflow_capped.size.should be == 20
+    @queue_overflow_capped.pop.should be == "message_10"
+  end
+  it 'should contains only unique items' do
+    20.times {@queue_overflow_uniq << "A"}
+    20.times {@queue_overflow_uniq << "B"}
+    @queue_overflow_uniq.size.should be == 2
+  end
+end

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,34 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require "digest/md5"
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+  config.mock_with :flexmock
+  config.around(:each) do |example|
+    VCR.use_cassette(Digest::MD5.hexdigest(example.metadata[:full_description])) do
+      example.run
+    end
+  end
+end
+require "vcr"
+require "polipus"
+VCR.configure do |c|
+  c.cassette_library_dir = "#{File.dirname(__FILE__)}/cassettes"
+  c.hook_into :webmock
+end
+def page_factory url, params = {}
+  Polipus::Page.new url, params
+end

data/spec/storage_mongo_spec.rb ADDED Viewed

@@ -0,0 +1,102 @@
+require "spec_helper"
+require "mongo"
+require "polipus/storage/mongo_store"
+describe Polipus::Storage::MongoStore do
+  before(:all)do
+    @mongo = Mongo::Connection.new("localhost", 27017, :pool_size => 15, :pool_timeout => 5).db('_test_polipus')
+    @mongo['_test_pages'].drop
+    @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+  end
+  after(:all) do
+    @mongo['_test_pages'].drop
+  end
+  after(:each) do
+    @mongo['_test_pages'].drop
+  end
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com', :code => 200, :body => '<html></html>'
+    uuid = @storage.add p
+    uuid.should be == 'ed646a3334ca891fd3467db131372140'
+    @storage.count.should be 1
+    @mongo['_test_pages'].count.should be 1
+    p = @storage.get p
+    p.url.to_s.should be == 'http://www.google.com'
+    p.body.should be == '<html></html>'
+  end
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    @storage.add p
+    p = @storage.get p
+    p.code.should be == 301
+    @mongo['_test_pages'].count.should be 1
+  end
+  it 'should iterate over stored pages' do
+    @storage.each do |k, page|
+      k.should be == "ed646a3334ca891fd3467db131372140"
+      page.url.to_s.should be == 'http://www.google.com'
+    end
+  end
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    @storage.remove p
+    @storage.get(p).should be_nil
+    @storage.count.should be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol', :code => 200, :body => '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', :code => 200, :body => '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    @storage.exists?(p_no_query).should be_true
+    @storage.remove p
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol', :code => 200, :body => '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', :code => 200, :body => '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    @storage.exists?(p_no_query).should be_true
+    @storage.remove p
+  end
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com',  :code => 200, :body => '<html></html>'
+    p.user_data.name = 'Test User Data'
+    @storage.add p
+    @storage.exists?(p).should be_true
+    p = @storage.get(p)
+    p.user_data.name.should be == 'Test User Data'
+    @storage.remove p
+  end
+  it 'should honor the except parameters' do
+    storage = Polipus::Storage.mongo_store(@mongo, '_test_pages', ['body'])
+    p = page_factory 'http://www.user-doo.com',  :code => 200, :body => '<html></html>'
+    storage.add p
+    p = storage.get p
+    p.body.should be_empty
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', :code => 200, :body => '<html></html>'
+    @storage.exists?(p_other).should be_false
+    @storage.add p_other
+    @storage.exists?(p_other).should be_true
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', :code => 200, :body => '<html></html>'
+    @storage.exists?(p_other).should be_true
+    @storage.include_query_string_in_uuid = true
+    @storage.exists?(p_other).should be_false
+  end
+end

data/spec/storage_s3_spec.rb ADDED Viewed

@@ -0,0 +1,115 @@
+require "spec_helper"
+require "aws/s3"
+require "polipus/storage/s3_store"
+describe Polipus::Storage::S3Store do
+  before(:each) do
+    @storage = Polipus::Storage.s3_store(
+      '_test_pages',
+      {
+        :access_key_id => 'XXXXXXX',
+        :secret_access_key => 'XXXX'
+      }
+    )
+  end
+  after(:each) {@storage.clear}
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com', :code => 200, :body => '<html></html>'
+    uuid = @storage.add p
+    uuid.should be == 'ed646a3334ca891fd3467db131372140'
+    @storage.count.should be 1
+    p = @storage.get p
+    p.url.to_s.should be == 'http://www.google.com'
+    p.body.should be == '<html></html>'
+    @storage.remove p
+  end
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    @storage.add p
+    p = @storage.get p
+    p.code.should be == 301
+    @storage.count.should be == 1
+    @storage.remove p
+  end
+  it 'should iterate over stored pages' do
+    10.times {|i| @storage.add page_factory("http://www.google.com/p_#{i}", :code => 200, :body => "<html>#{i}</html>")}
+    @storage.count.should be 10
+    @storage.each do |k, page|
+      k.should be =~ /[a-f0-9]{32}/
+    end
+  end
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', :code => 301, :body => '<html></html>'
+    @storage.add p
+    @storage.remove p
+    @storage.get(p).should be_nil
+    @storage.count.should be 0
+  end
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol', :code => 200, :body => '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', :code => 200, :body => '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    @storage.exists?(p_no_query).should be_true
+    @storage.remove p
+  end
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol', :code => 200, :body => '<html></html>'
+    p_no_query = page_factory 'http://www.asd.com', :code => 200, :body => '<html></html>'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    @storage.exists?(p_no_query).should be_true
+    @storage.remove p
+  end
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com',  :code => 200, :body => '<html></html>'
+    p.user_data.name = 'Test User Data'
+    @storage.add p
+    @storage.exists?(p).should be_true
+    p = @storage.get(p)
+    p.user_data.name.should be == 'Test User Data'
+    @storage.remove p
+  end
+  it 'should honor the except parameters' do
+    storage = storage = Polipus::Storage.s3_store(
+        '_test_pages',
+         {
+          :access_key_id => 'XXXXXXX',
+          :secret_access_key => 'XXXX'
+        },
+        ['body']
+      )
+    p = page_factory 'http://www.user-doo.com',  :code => 200, :body => '<html></html>'
+    storage.add p
+    p = storage.get p
+    p.body.should be_nil
+    storage.clear
+  end
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com', :code => 200, :body => '<html></html>'
+    @storage.exists?(p_other).should be_false
+    @storage.add p_other
+    @storage.exists?(p_other).should be_true
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol', :code => 200, :body => '<html></html>'
+    @storage.exists?(p_other).should be_true
+    @storage.include_query_string_in_uuid = true
+    @storage.exists?(p_other).should be_false
+    @storage.remove p_other
+  end
+end