parallel588_polipus 0.4.0

data/spec/polipus_spec.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+require 'spec_helper'
+
+describe Polipus::PolipusCrawler do
+  after(:each) { Redis.new(db: 10).flushdb }
+  let(:p_options) do
+    {
+      workers: 1,
+      redis_options: { host: 'localhost', db: 10 },
+      depth_limit: 1,
+      queue_timeout: 1,
+      user_agent: 'polipus-rspec',
+      logger: logger,
+      logger_level: Logger::DEBUG,
+      storage: Polipus::Storage.memory_store
+    }
+  end
+  let(:polipus) do
+    Polipus::PolipusCrawler.new('polipus-rspec', ['http://rubygems.org/gems'], p_options)
+  end
+
+  let(:init_page)do
+    Polipus::Page.new 'http://rubygems.org/gems'
+  end
+
+  let(:logger) { Logger.new(nil) }
+
+  context 'polipus' do
+    it 'should create a polipus instance' do
+      expect(polipus).to be_an_instance_of Polipus::PolipusCrawler
+    end
+
+    it 'should execute a crawling session' do
+      polipus.takeover
+      expect(polipus.storage.exists?(init_page)).to be_truthy
+      expect(polipus.storage.get(init_page).links.count).to be polipus.storage.count
+    end
+
+    it 'should filter unwanted urls' do
+      polipus.skip_links_like(/\/pages\//)
+      polipus.takeover
+      expect(polipus.storage.get(init_page).links
+        .reject { |e| e.path.to_s =~ /\/pages\// }.count).to be polipus.storage.count
+    end
+
+    it 'should follow only wanted urls' do
+      polipus.follow_links_like(/\/pages\//)
+      polipus.follow_links_like(/\/gems$/)
+      polipus.takeover
+      expect(polipus.storage.get(init_page).links
+        .reject { |e| ![/\/pages\//, /\/gems$/].any? { |p| e.path =~ p } }
+        .count).to be polipus.storage.count
+    end
+
+    it 'should refresh expired pages' do
+      polipus.ttl_page = 3600
+      polipus.takeover
+      polipus.storage.each do |_id, page|
+        page.fetched_at = page.fetched_at - 3600
+        polipus.storage.add(page)
+      end
+      polipus.storage.each { |_id, page| expect(page.expired?(3600)).to be_truthy }
+      polipus.takeover
+      polipus.storage.each { |_id, page| expect(page.expired?(3600)).to be_falsey }
+    end
+
+    it 'should re-download seeder urls no matter what' do
+      cache_hit = {}
+      polipus.follow_links_like(/\/gems$/)
+      polipus.on_page_downloaded do |page|
+        cache_hit[page.url.to_s] ||= 0
+        cache_hit[page.url.to_s] += 1
+      end
+      polipus.takeover
+      polipus.takeover
+      expect(cache_hit['http://rubygems.org/gems']).to be 2
+    end
+
+    it 'should call on_page_error code blocks when a page has error' do
+      p = Polipus::PolipusCrawler.new('polipus-rspec', ['http://dasd.adad.dom/'], p_options.merge(open_timeout: 1, read_timeout: 1))
+      a_page = nil
+      p.on_page_error { |page| a_page = page }
+      p.takeover
+      expect(a_page).not_to be_nil
+      expect(a_page.error).not_to be_nil
+    end
+
+    it 'should obey to the robots.txt file' do
+      lopt = p_options
+      lopt[:obey_robots_txt] = true
+      polipus = Polipus::PolipusCrawler.new('polipus-rspec', ['https://rubygems.org/gems/polipus'], lopt)
+      polipus.depth_limit = 1
+      polipus.takeover
+      polipus.storage.each { |_id, page| expect(page.url.path =~ /$\/downloads\//).to be_falsey }
+    end
+
+    it 'should obey to the robots.txt file with list user_agent' do
+      user_agent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)"
+      lopt = p_options
+      lopt[:obey_robots_txt] = true
+      lopt[:user_agent] = [user_agent]
+      flexmock(Polipus::Robotex).should_receive(:new).with(user_agent)
+      Polipus::PolipusCrawler.new('polipus-rspec', ['https://rubygems.org/gems/polipus'], lopt)
+
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,42 @@
+# Require this file using `require "spec_helper"`
+# to ensure that it is only loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require 'digest/md5'
+require 'coveralls'
+require 'vcr'
+require 'webmock/rspec'
+
+Coveralls.wear!
+
+VCR.configure do |c|
+  c.cassette_library_dir = "#{File.dirname(__FILE__)}/cassettes"
+  c.hook_into :webmock
+end
+
+require 'polipus'
+
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+  config.mock_with :flexmock
+  config.around(:each) do |example|
+    t = Time.now
+    print example.metadata[:full_description]
+    VCR.use_cassette(Digest::MD5.hexdigest(example.metadata[:full_description])) do
+      example.run
+      puts " [#{Time.now - t}s]"
+    end
+  end
+  config.before(:each) { Polipus::SignalHandler.disable }
+end
+
+def page_factory(url, params = {})
+  Polipus::Page.new url, params
+end

metadata

@@ -0,0 +1,348 @@
+--- !ruby/object:Gem::Specification
+name: parallel588_polipus
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- Francesco Laurita
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: http-cookie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.4
+- !ruby/object:Gem::Dependency
+  name: hiredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.5
+- !ruby/object:Gem::Dependency
+  name: redis-queue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+- !ruby/object:Gem::Dependency
+  name: redis-bloomfilter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: rethinkdb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.20.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.20.0
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: "\n    An easy to use distributed web-crawler framework based on Redis\n
+  \ "
+email:
+- francesco.laurita@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".document"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".travis.yml"
+- AUTHORS.md
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/basic.rb
+- examples/error_handling.rb
+- examples/incremental.rb
+- examples/robots_txt_handling.rb
+- examples/survival.rb
+- lib/polipus.rb
+- lib/polipus/http.rb
+- lib/polipus/page.rb
+- lib/polipus/plugin.rb
+- lib/polipus/plugins/cleaner.rb
+- lib/polipus/plugins/sample.rb
+- lib/polipus/plugins/sleeper.rb
+- lib/polipus/queue_overflow.rb
+- lib/polipus/queue_overflow/base.rb
+- lib/polipus/queue_overflow/dev_null_queue.rb
+- lib/polipus/queue_overflow/manager.rb
+- lib/polipus/queue_overflow/mongo_queue.rb
+- lib/polipus/queue_overflow/mongo_queue_capped.rb
+- lib/polipus/queue_overflow/worker.rb
+- lib/polipus/robotex.rb
+- lib/polipus/signal_handler.rb
+- lib/polipus/storage.rb
+- lib/polipus/storage/base.rb
+- lib/polipus/storage/dev_null.rb
+- lib/polipus/storage/memory_store.rb
+- lib/polipus/storage/mongo_store.rb
+- lib/polipus/storage/rethink_store.rb
+- lib/polipus/url_tracker.rb
+- lib/polipus/url_tracker/bloomfilter.rb
+- lib/polipus/url_tracker/redis_set.rb
+- lib/polipus/version.rb
+- polipus.gemspec
+- spec/cassettes/11c3eb8bf35dfc179dc5ce44f6f5f458.yml
+- spec/cassettes/1f6e1d7743ecaa86594b4e68a6462689.yml
+- spec/cassettes/6adfecdb274dd26ffd3713169583ca91.yml
+- spec/cassettes/978ac0eeb5df63a019b754cc8a965b06.yml
+- spec/cassettes/b389efd1dcb8f09393b5aae1627c2a83.yml
+- spec/cassettes/bc6fb220895689be7eeb05b09969a18d.yml
+- spec/cassettes/c5ce68499027d490adfbb6e5541881e4.yml
+- spec/cassettes/ce16b11a7df0b70fe90c7f90063fdb8c.yml
+- spec/cassettes/gzipped_on.yml
+- spec/cassettes/http_cookies.yml
+- spec/cassettes/http_tconnection_max_hits.yml
+- spec/cassettes/http_test.yml
+- spec/cassettes/http_test_redirect.yml
+- spec/clear.rb
+- spec/polipus/http_spec.rb
+- spec/polipus/page_spec.rb
+- spec/polipus/queue_overflow/manager_spec.rb
+- spec/polipus/queue_overflow_spec.rb
+- spec/polipus/robotex_spec.rb
+- spec/polipus/signal_handler_spec.rb
+- spec/polipus/storage/memory_store_spec.rb
+- spec/polipus/storage/mongo_store_spec.rb
+- spec/polipus/storage/rethink_store_spec.rb
+- spec/polipus/url_tracker_spec.rb
+- spec/polipus_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/taganaka/polipus
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: polipus
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Polipus distributed web-crawler framework
+test_files: []
+has_rdoc: