parallel588_polipus 0.4.0

data/spec/cassettes/http_test_redirect.yml

@@ -0,0 +1,71 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 300
+      message: Multiple Choices
+    headers:
+      Date:
+      - Mon, 10 Jun 2013 08:58:25 GMT
+      Server:
+      - Apache/2.2.22 (Ubuntu)
+      Cache-Control:
+      - no-cache
+      Location:
+      - http://greenbytes.de/tech/tc/httpredirects/300.txt
+      Content-Length:
+      - '27'
+    body:
+      encoding: US-ASCII
+      string: ! '300 Redirect Response Body
+
+'
+    http_version: 
+  recorded_at: Mon, 10 Jun 2013 08:58:25 GMT
+- request:
+    method: get
+    uri: http://greenbytes.de/tech/tc/httpredirects/300.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 10 Jun 2013 08:58:25 GMT
+      Server:
+      - Apache/2.2.22 (Ubuntu)
+      Last-Modified:
+      - Tue, 08 Jan 2013 17:31:05 GMT
+      Etag:
+      - ! '"b8306c-31-4d2ca4f7df2ca"'
+      Accept-Ranges:
+      - bytes
+      Content-Length:
+      - '49'
+      Content-Type:
+      - text/plain
+    body:
+      encoding: US-ASCII
+      string: ! "You have reached the target\r\nof a 300 redirect.\r\n"
+    http_version: 
+  recorded_at: Mon, 10 Jun 2013 08:58:25 GMT
+recorded_with: VCR 2.5.0

data/spec/clear.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+require 'yaml'
+Dir.glob('./cassettes/*.yml').each do|f|
+  next unless f =~ /[a-f0-9]{32}/
+  d = YAML.load_file(f)
+  d['http_interactions'].each do |r|
+    r['request'].delete('headers')
+    r['response'].delete('headers')
+  end
+  File.open(f, 'w') { |fw| fw.write(d.to_yaml) }
+  # puts d.to_yaml
+end

data/spec/polipus/http_spec.rb

@@ -0,0 +1,139 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'mongo'
+require 'polipus/http'
+require 'polipus/page'
+
+describe Polipus::HTTP do
+  it 'should download a page' do
+    VCR.use_cassette('http_test') do
+      http = Polipus::HTTP.new
+      page = http.fetch_page('http://sfbay.craigslist.org/apa/')
+      expect(page).to be_an_instance_of(Polipus::Page)
+      expect(page.doc.search('title').text.strip).to eq 'SF bay area apts/housing for rent classifieds  - craigslist'
+      expect(page.fetched_at).not_to be_nil
+      expect(page.fetched?).to be_truthy
+    end
+  end
+
+  it 'should follow a redirect' do
+    VCR.use_cassette('http_test_redirect') do
+      http = Polipus::HTTP.new
+      page = http.fetch_page('http://greenbytes.de/tech/tc/httpredirects/t300bodyandloc.asis')
+
+      expect(page).to be_an_instance_of(Polipus::Page)
+      expect(page.code).to be 200
+      expect(page.url.to_s).to eq 'http://greenbytes.de/tech/tc/httpredirects/300.txt'
+      expect(page.body.strip).to eq "You have reached the target\r\nof a 300 redirect."
+    end
+  end
+
+  describe 'proxy settings' do
+    it 'should set proxy correctly using a procedure' do
+      http = Polipus::HTTP.new(proxy_host: -> _con { '127.0.0.0' }, proxy_port: -> _con { 8080 })
+      expect(http.proxy_host).to eq '127.0.0.0'
+      expect(http.proxy_port).to be 8080
+    end
+
+    it 'should set proxy correctly using shorthand method' do
+      http = Polipus::HTTP.new(proxy_host_port: -> _con { ['127.0.0.0', 8080] })
+      expect(http.proxy_host_port).to eq ['127.0.0.0', 8080]
+    end
+
+    it 'should set proxy w/ auth correctly using shorthand method' do
+      http = Polipus::HTTP.new(proxy_host_port: -> _con { ['127.0.0.0', 8080, 'a', 'b'] })
+      expect(http.proxy_host_port).to eq ['127.0.0.0', 8080, 'a', 'b']
+    end
+
+    it 'should set proxy settings' do
+      http = Polipus::HTTP.new(proxy_host: '127.0.0.0', proxy_port:  8080, proxy_user: 'a', proxy_pass: 'b')
+      expect(http.proxy_port).to be 8080
+      expect(http.proxy_host).to eq '127.0.0.0'
+      expect(http.proxy_user).to eq 'a'
+      expect(http.proxy_pass).to eq 'b'
+    end
+  end
+
+  describe 'compressed content handling' do
+    it 'should decode gzip content' do
+      VCR.use_cassette('gzipped_on') do
+        http = Polipus::HTTP.new(logger: Logger.new(STDOUT))
+        page = http.fetch_page('http://www.whatsmyip.org/http-compression-test/')
+        expect(page.doc.css('.gzip_yes')).not_to be_empty
+      end
+    end
+
+    it 'should decode deflate content' do
+      http = Polipus::HTTP.new(logger: Logger.new(STDOUT))
+      page = http.fetch_page('http://david.fullrecall.com/browser-http-compression-test?compression=deflate-http')
+      expect(page.headers.fetch('content-encoding').first).to eq 'deflate'
+      expect(page.body.include?('deflate-http')).to be_truthy
+    end
+  end
+
+  describe 'staled connections' do
+    it 'should refresh a staled connection' do
+      VCR.use_cassette('http_tconnection_max_hits') do
+        http = Polipus::HTTP.new(connection_max_hits: 1, logger: Logger.new(STDOUT))
+        http.class.__send__(:attr_reader, :connections)
+        http.class.__send__(:attr_reader, :connections_hits)
+        http.fetch_page('https://www.yahoo.com/')
+        expect(http.connections['www.yahoo.com'][443]).not_to be_nil
+        old_conn = http.connections['www.yahoo.com'][443]
+        expect(http.connections_hits['www.yahoo.com'][443]).to be 1
+
+        http.fetch_page('https://www.yahoo.com/tech/expectant-parents-asked-the-internet-to-name-their-83416450388.html')
+        expect(http.connections_hits['www.yahoo.com'][443]).to be 1
+        expect(http.connections['www.yahoo.com'][443]).not_to be old_conn
+      end
+    end
+  end
+
+  describe 'cookies' do
+    it 'should handle cookies correctly' do
+      VCR.use_cassette('http_cookies') do
+        http = Polipus::HTTP.new(accept_cookies: true)
+        http.fetch_page 'http://www.whatarecookies.com/cookietest.asp'
+        expect(http.accept_cookies?).to be_truthy
+        expect(http.cookie_jar.cookies(URI('http://www.whatarecookies.com/cookietest.asp'))).not_to be_empty
+      end
+    end
+  end
+
+  describe 'net errors' do
+    it 'should handle net errors correctly' do
+      VCR.use_cassette('http_errors') do
+        http = Polipus::HTTP.new(open_timeout: 1, read_timeout: 1)
+        expect(http.fetch_page('http://www.wrong-domain.lol/').error).not_to be_nil
+      end
+    end
+  end
+
+  describe 'random user_agent' do
+    context 'when user_agent is string' do
+      it '#user_agent' do
+        http = Polipus::HTTP.new(open_timeout: 1, read_timeout: 1, user_agent: 'Googlebot')
+        expect(http.user_agent).to eq('Googlebot')
+     end
+    end
+
+    context 'when user_agent is list' do
+      let(:user_agents) {
+        ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",
+         "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
+         "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
+         "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre",
+         "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10",
+         "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)",
+         "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 GTB5",
+         "Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)"
+        ]
+      }
+
+      it '#user_agent' do
+        http = Polipus::HTTP.new(open_timeout: 1, read_timeout: 1, user_agent: user_agents)
+        expect(user_agents).to include(http.user_agent)
+      end
+    end
+  end
+end

data/spec/polipus/page_spec.rb

@@ -0,0 +1,68 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'polipus/page'
+
+describe Polipus::Page do
+  let(:page) do
+    body = <<EOF
+    <html>
+      <body>
+        <a href="/page/1">1</a>
+        <a href="/page/2">2</a>
+        <a href="http://www.google.com/page/3">3</a>
+        <a href="http://google.com/page/3">4</a>
+        <a href="http://not.google.com/page/3">4</a>
+      </body>
+    </html>
+EOF
+    Polipus::Page.new 'http://www.google.com/',
+                      code: 200,
+                      body: body,
+                      headers: { 'content-type' => ['text/html'] },
+                      domain_aliases: %w(www.google.com google.com)
+  end
+
+  it 'should be fetched' do
+    expect(page.fetched?).to be_truthy
+  end
+
+  it 'should honor domain_aliases attribute' do
+    expect(page.links.count).to be 4
+  end
+
+  context 'page expiring' do
+    let(:page) do
+      Polipus::Page.new 'http://www.google.com/',
+                        code: 200,
+                        body: '',
+                        headers: { 'content-type' => ['text/html'] },
+                        domain_aliases: %w(www.google.com google.com),
+                        fetched_at: (Time.now.to_i - 30)
+    end
+
+    it 'should be marked at expired' do
+      expect(page.expired?(20)).to be_truthy
+    end
+
+    it 'should NOT be marked at expired' do
+      expect(page.expired?(60)).to be_falsey
+    end
+  end
+
+  context 'page error' do
+    let(:page) do
+      Polipus::Page.new 'http://www.google.com/', error: 'an error'
+    end
+
+    it 'should serialize an error' do
+      expect(page.to_hash['error']).to eq 'an error'
+    end
+  end
+
+  context 'page code' do
+    it 'should identify HTTPSuccess code' do
+      expect(Polipus::Page.new('http://www.google.com/', code: 201).success?).to be_truthy
+      expect(Polipus::Page.new('http://www.google.com/', code: 404).success?).to be_falsey
+    end
+  end
+end

data/spec/polipus/queue_overflow/manager_spec.rb

@@ -0,0 +1,88 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'mongo'
+require 'polipus/queue_overflow'
+require 'redis-queue'
+
+describe Polipus::QueueOverflow::Manager do
+  before(:all) do
+    @mongo = Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('_test_polipus')
+    @mongo['_test_pages'].drop
+    @storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
+    @redis_q = Redis::Queue.new('queue_test', 'bp_queue_test', redis: Redis.new)
+    @queue_overflow   = Polipus::QueueOverflow.mongo_queue(nil, 'queue_test')
+    @redis = Redis.new
+    @polipus = flexmock('polipus')
+    @polipus.should_receive(:queue_overflow_adapter).and_return(@queue_overflow)
+    @polipus.should_receive(:storage).and_return(@storage)
+    @polipus.should_receive(:redis).and_return(@redis)
+    @polipus.should_receive(:job_name).and_return('___test')
+    @polipus.should_receive(:logger).and_return(Logger.new(nil))
+    @manager = Polipus::QueueOverflow::Manager.new(@polipus, @redis_q, 10)
+  end
+
+  before(:each) do
+    @queue_overflow.clear
+    @redis_q.clear
+    @storage.clear
+  end
+
+  after(:all) do
+    @queue_overflow.clear
+    @redis_q.clear
+  end
+
+  it 'should remove 10 items' do
+    expect(@manager.perform).to eq([0, 0])
+    20.times { |i| @redis_q << page_factory("http://www.user-doo.com/page_#{i}",  code: 200, body: '<html></html>').to_json  }
+    expect(@manager.perform).to eq([10, 0])
+    expect(@queue_overflow.size).to eq(10)
+    expect(@redis_q.size).to eq(10)
+  end
+
+  it 'should restore 10 items' do
+    expect(@manager.perform).to eq([0, 0])
+    10.times { |i| @queue_overflow << page_factory("http://www.user-doo-bla.com/page_#{i}",  code: 200, body: '<html></html>').to_json }
+    expect(@manager.perform).to eq([0, 10])
+    expect(@queue_overflow.size).to eq(0)
+    expect(@redis_q.size).to eq(10)
+    expect(@manager.perform).to eq([0, 0])
+  end
+
+  it 'should restore 3 items' do
+    expect(@manager.perform).to eq([0, 0])
+    3.times { |i| @queue_overflow << page_factory("http://www.user-doo-bu.com/page_#{i}",  code: 200, body: '<html></html>').to_json }
+    expect(@manager.perform).to eq([0, 3])
+    expect(@queue_overflow.size).to eq(0)
+    expect(@redis_q.size).to eq(3)
+    expect(@manager.perform).to eq([0, 0])
+  end
+
+  it 'should restore 0 items' do
+    expect(@manager.perform).to eq([0, 0])
+    10.times do|i|
+      p = page_factory("http://www.user-doo-bu.com/page_#{i}",  code: 200, body: '<html></html>')
+      @storage.add p
+      @queue_overflow << p.to_json
+    end
+    expect(@manager.perform).to eq([0, 0])
+    expect(@queue_overflow.size).to eq(0)
+    expect(@redis_q.size).to eq(0)
+    expect(@manager.perform).to eq([0, 0])
+  end
+
+  it 'should filter an url based on the spec' do
+    @queue_overflow.clear
+    @redis_q.clear
+    10.times { |i| @queue_overflow << page_factory("http://www.user-doo.com/page_#{i}",  code: 200, body: '<html></html>').to_json  }
+    @manager.url_filter do |page|
+      page.url.to_s.end_with?('page_0') ? false : true
+    end
+    expect(@manager.perform).to eq([0, 9])
+    expect(@queue_overflow.size).to eq(0)
+    expect(@redis_q.size).to eq(9)
+    @manager.url_filter do |_page|
+      true
+    end
+  end
+end

data/spec/polipus/queue_overflow_spec.rb

@@ -0,0 +1,66 @@
+# encoding: UTF-8
+require 'spec_helper'
+require 'polipus/queue_overflow'
+
+describe Polipus::QueueOverflow do
+  before(:all) do
+    @queue_overflow        = Polipus::QueueOverflow.mongo_queue(nil, 'queue_test')
+    @queue_overflow_capped = Polipus::QueueOverflow.mongo_queue_capped(nil, 'queue_test_c', max: 20)
+    @queue_overflow_uniq   = Polipus::QueueOverflow.mongo_queue(nil, 'queue_test_u', ensure_uniq: true)
+  end
+
+  before(:each) do
+    @queue_overflow.clear
+    @queue_overflow_capped.clear
+    @queue_overflow_uniq.clear
+  end
+
+  after(:all) do
+    @queue_overflow.clear
+    @queue_overflow_uniq.clear
+    @queue_overflow_capped.clear
+  end
+
+  it 'should work' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      expect(q.empty?).to be_truthy
+      expect(q.pop).to be_nil
+      q << 'test'
+      expect(q.size).to eq(1)
+      expect(q.pop).to eq('test')
+      expect(q.empty?).to be_truthy
+      expect(q.pop).to be_nil
+      expect(q.size).to eq(0)
+      expect(q.empty?).to be_truthy
+    end
+  end
+
+  it 'should act as a queue' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      10.times { |i| q << "message_#{i}" }
+      expect(q.size).to eq(10)
+      expect(q.pop).to eq('message_0')
+    end
+  end
+
+  it 'should work with complex paylod' do
+    [@queue_overflow, @queue_overflow_capped, @queue_overflow_uniq].each do |q|
+      a = { 'a' => [1, 2, 3], 'b' => 'a_string' }
+      q << a.to_json
+      b = q.pop
+      expect(JSON.parse(b)).to eq(a)
+    end
+  end
+
+  it 'should honor max items if it is capped' do
+    30.times { |i| @queue_overflow_capped << "message_#{i}" }
+    expect(@queue_overflow_capped.size).to eq(20)
+    expect(@queue_overflow_capped.pop).to eq('message_10')
+  end
+
+  it 'should contains only unique items' do
+    20.times { @queue_overflow_uniq << 'A' }
+    20.times { @queue_overflow_uniq << 'B' }
+    expect(@queue_overflow_uniq.size).to eq(2)
+  end
+end