polipus 0.3.7 → 0.4.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZmMzMzJiMDQ1NGQzYTRhNTRmMGZiZWNhY2Y4YTgwNTU2OTljNzdkNQ==
+    NGU0MWZlMWIwMGM2MWNhMmJiOTU3MDM0YWI0ZjY0MzI3NmNkYzgyNA==
   data.tar.gz: !binary |-
-    ZDQyNTBiNjkzYzQ0ZDc3ODc3N2MxMTcxOTAzOWFkZjliZGZlYzFjZQ==
+    MTAxYjZmYWNjMTlkYzk3Y2ZhNjdjMDFmODM2OTQ5YmQ3ZDcyNTQzNw==
 SHA512:
   metadata.gz: !binary |-
-    MWMwNjYzMzY5ZmYzYzNiMTdlMTFkOTNhNGYzNTViZmQ4ZjdkYWFkMjE1YTMz
-    MDc3ZDA1Y2JlN2FiNjVmZGZmYWRlNDU2YjcwYjdlNjFiYjViY2NiZTA1OTNm
-    M2JhYjVhYjkwMzlhNWIxMmM2MzhmNWM5YmY5ZTQ1ZjE2OWY5MGM=
+    ZDE5NDExNmE1M2M4MDM0MTExNzVkNmM0ZmVjNTQyOTg5Y2JkYTdmZjBhY2Fi
+    NmM1MDc5ZTI0ODkxNzlhY2ZkZGVjZjI2ZTMyNjZhNTMwOWZhZjQ2OGY4ZTJl
+    ZmI4ZjUwNmQ3YjAxOTdlODI2MWRmMWUzNjY3Yzk1MmNiZTk0Y2E=
   data.tar.gz: !binary |-
-    ODlkYzUyNTI1MTVmOTI1YzI2ZjFkMzY1NTMyZWIzMWEwYjI4MGFmODc4Y2Yx
-    ZGVkZTA2NmM2OGU5ZThkOTg1ODJjYjMxNTJiNGZkZDVhYzkwZjkzOWI1ZGI2
-    OTcxOGViODEwZDBiYmQ1ZDU4NmRmZWI2NWZkMTFlNzIzYjdlMjc=
+    YjNlYTZlMjU5OTBmN2ZjMTI1OTY1MzgzNzBhYjBjYTAzN2MwYzY0M2U1ZGYw
+    MjcyMmMxM2I1ZTY5OWEzZDQ3MjQ1OTgzMzcyN2I5NGQ5NTAzZTEzY2FmMDRl
+    MmJiZDdjOTFmMjQwZWU1MzM1OGJhN2E4NTRhNGExZTAyMTVmN2I=

data/.rspec

@@ -1,2 +1,2 @@
 --color
---format progress
+--format documentation

data/.rubocop.yml

@@ -4,14 +4,14 @@ AllCops:
    - my_test/**/*
    - examples/**/*
 
-Style/LineLength:
+Metrics/LineLength:
   Enabled: false
 
 Style/TrivialAccessors:
   Enabled: false
 
-Style/ClassLength:
+Metrics/ClassLength:
   Enabled: false
 
-Style/MethodLength:
+Metrics/MethodLength:
   Enabled: false

data/.rubocop_todo.yml

@@ -10,7 +10,7 @@ Style/ClassVars:
   Enabled: false
 
 # Offense count: 10
-Style/CyclomaticComplexity:
+Metrics/CyclomaticComplexity:
   Max: 16
 
 # Offense count: 26

data/.travis.yml

@@ -3,10 +3,20 @@ rvm:
   - jruby
   - 1.9.3
   - 2.0.0
-  - 2.1.2
+  - 2.1.5
+  - 2.2.0
   - rbx-2
 
-services: 
-  - mongodb
-  - redis
+# Until travis supports rethinkdb as service...
+before_install:
+  - source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list
+  - wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add -
+  - sudo apt-get update -q
+  - sudo apt-get install rethinkdb
+  - sudo cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/instance1.conf
+  - sudo service rethinkdb restart
 
+services:
+  - redis
+  - mongodb
+#  - rethinkdb

data/AUTHORS.md

@@ -2,3 +2,4 @@
 
 * [Francesco Laurita](francesco.laurita@gmail.com)
 * [Tobias L. Maier](http://tobiasmaier.info/)
+* [Marcos Piccinini](https://github.com/nofxx)

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.4.0 (2015-01-12)
+
+[Compare changes in gem](https://github.com/taganaka/polipus/compare/0.3.3...0.4.0)
+
+* Adds RethinkDB Storage
+* BugFix: Update and fix mongo driver v1.11.1 'upsert: 1' -> 'upsert: true'
+* Organize and update specs to rspec 3
+
 ## 0.3.3 (2015-06-26)
 
 [Compare changes in gem](https://github.com/taganaka/polipus/compare/0.3.2...0.3.3)
@@ -27,7 +35,7 @@
     ```ruby
     enable_signal_handler: true / false
     ```
-  
+
 * Zlib::GzipFile::Error handling
   [da3b927](https://github.com/taganaka/polipus/commit/da3b927acb1b50c26276ed458da0a365c22fd98b)
 * Faster and easier overflow management

data/Gemfile

@@ -1,3 +1,12 @@
 source 'https://rubygems.org'
 
 gemspec
+
+platform :ruby do
+  gem 'bson_ext'
+end
+
+platform :jruby do
+  gem 'json'
+  gem 'bson'
+end

data/README.md

@@ -1,9 +1,8 @@
 [![Build Status](https://travis-ci.org/taganaka/polipus.svg?branch=master)](https://travis-ci.org/taganaka/polipus)
-[![Coverage Status](https://coveralls.io/repos/taganaka/polipus/badge.png?branch=master)](https://coveralls.io/r/taganaka/polipus?branch=master)
-[![Code Climate](https://codeclimate.com/github/taganaka/polipus.png)](https://codeclimate.com/github/taganaka/polipus)
+[![Coverage Status](https://img.shields.io/coveralls/taganaka/polipus/master.svg)](https://coveralls.io/r/taganaka/polipus?branch=master)
+[![Code Climate](https://codeclimate.com/github/taganaka/polipus.svg)](https://codeclimate.com/github/taganaka/polipus)
 [![RubyGems](http://img.shields.io/gem/v/polipus.svg)](https://rubygems.org/gems/polipus)
 
-
 # Polipus #
 
 A distributed web crawler written in ruby, backed by Redis

data/Rakefile

@@ -2,9 +2,7 @@
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = 'spec/*_spec.rb'
-end
+RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
 task test: :spec

data/examples/basic.rb

@@ -20,7 +20,14 @@ options = {
   user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71',
   # Use 5 threads
   workers: 5,
-  # Logs goes to the crawler.log file
+  # Queue overflow settings:
+  #  * No more than 5000 elements on the Redis queue
+  #  * Exceeded Items will stored on Mongo into 'rubygems_queue_overflow' collection
+  #  * Check cycle is done every 60 sec
+  queue_items_limit: 5_000,
+  queue_overflow_adapter: Polipus::QueueOverflow.mongo_queue(mongo, 'rubygems_queue_overflow'),
+  queue_overflow_manager_check_time: 60,
+  # Logs goes to the stdout
   logger: Logger.new(STDOUT)
 }
 Polipus::Plugin.register Polipus::Plugin::Cleaner, reset: true

data/lib/polipus.rb

@@ -118,6 +118,7 @@ module Polipus
       @on_before_save     = []
       @on_page_error      = []
       @focus_crawl_block  = nil
+      @on_crawl_start     = []
       @on_crawl_end       = []
       @redis_factory      = nil
 
@@ -131,6 +132,20 @@ module Polipus
       @robots = Polipus::Robotex.new(@options[:user_agent]) if @options[:obey_robots_txt]
       # Attach signal handling if enabled
       SignalHandler.enable if @options[:enable_signal_handler]
+
+      if queue_overflow_adapter
+        @on_crawl_start << lambda do |_|
+          Thread.new do
+            Thread.current[:name] = :overflow_items_controller
+            overflow_items_controller.run
+          end
+        end
+      end
+
+      @on_crawl_end << lambda do |_|
+        Thread.list.select { |thread| thread.status && Thread.current[:name] == :overflow_items_controller }.each(&:kill)
+      end
+
       execute_plugin 'on_initialize'
 
       yield self if block_given?
@@ -141,13 +156,13 @@ module Polipus
     end
 
     def takeover
-      overflow_items_controller if queue_overflow_adapter
-
       @urls.each do |u|
         add_url(u) { |page| page.user_data.p_seeded = true }
       end
       return if internal_queue.empty?
 
+      @on_crawl_start.each { |e| e.call(self) }
+
       execute_plugin 'on_crawl_start'
       @options[:workers].times do |worker_number|
         @workers_pool << Thread.new do
@@ -237,6 +252,7 @@ module Polipus
           end
         end
       end
+
       @workers_pool.each { |w| w.join }
       @on_crawl_end.each { |e| e.call(self) }
       execute_plugin 'on_crawl_end'
@@ -269,6 +285,12 @@ module Polipus
       self
     end
 
+    # A block of code will be executed when crawl session is starting
+    def on_crawl_start(&block)
+      @on_crawl_start << block
+      self
+    end
+
     # A block of code will be executed on every page downloaded
     # before being saved in the registered storage
     def on_before_save(&block)
@@ -439,17 +461,7 @@ module Polipus
         should_be_visited?(page.url, false)
       end
 
-      Thread.new do
-
-        loop do
-          @logger.info { 'Overflow Manager: cycle started' }
-          removed, restored = @overflow_manager.perform
-          @logger.info { "Overflow Manager: items removed=#{removed}, items restored=#{restored}, items stored=#{queue_overflow_adapter.size}" }
-          sleep @options[:queue_overflow_manager_check_time]
-          break if SignalHandler.terminated?
-        end
-
-      end
+      QueueOverflow::Worker.new(@overflow_manager)
     end
 
     def internal_queue

data/lib/polipus/queue_overflow.rb

@@ -1,5 +1,6 @@
 # encoding: UTF-8
 require 'polipus/queue_overflow/manager'
+require 'polipus/queue_overflow/worker'
 module Polipus
   module QueueOverflow
     def self.mongo_queue(mongo_db, queue_name, options = {})

data/lib/polipus/queue_overflow/manager.rb

@@ -3,6 +3,7 @@ module Polipus
   module QueueOverflow
     class Manager
       attr_accessor :url_filter
+      attr_reader :polipus
       def initialize(polipus, main_q, item_limit)
         @polipus = polipus
         @main_q  = main_q

data/lib/polipus/queue_overflow/mongo_queue.rb

@@ -28,7 +28,7 @@ module Polipus
 
       def push(data)
         if @options[:ensure_uniq]
-          @mongo_db[@collection_name].update({ payload: data }, { payload: data }, { upsert: 1, w: 1 })
+          @mongo_db[@collection_name].update({ payload: data }, { payload: data }, { upsert: true, w: 1 })
         else
           @mongo_db[@collection_name].insert(payload: data)
         end

data/lib/polipus/queue_overflow/worker.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+module Polipus
+  module QueueOverflow
+    class Worker
+      def initialize(manager)
+        @logger  = manager.polipus.logger
+        @delay   = manager.polipus.options[:queue_overflow_manager_check_time]
+        @adapter = manager.polipus.queue_overflow_adapter
+        @manager = manager
+      end
+
+      def run
+        @logger.info { 'Overflow::Worker::run' }
+        loop do
+          @logger.info { 'Overflow Manager: cycle started' }
+          removed, restored = @manager.perform
+          @logger.info { "Overflow Manager: items removed=#{removed}, items restored=#{restored}, items stored=#{@adapter.size}" }
+          sleep @delay
+          break if SignalHandler.terminated?
+        end
+      end
+    end
+  end
+end

data/lib/polipus/storage.rb

@@ -1,27 +1,21 @@
-# encoding: UTF-8
 require 'polipus/storage/base'
+
 module Polipus
   module Storage
-    def self.mongo_store(mongo = nil, collection_name = 'pages', except = [])
+    COLLECTION = 'pages'
+
+    def self.mongo_store(mongo = nil, collection = COLLECTION, except = [])
       require 'polipus/storage/mongo_store'
       mongo ||= Mongo::Connection.new('localhost', 27_017, pool_size: 15, pool_timeout: 5).db('polipus')
       fail 'First argument must be an instance of Mongo::DB' unless mongo.is_a?(Mongo::DB)
-      self::MongoStore.new(mongo: mongo, collection: collection_name, except: except)
+      self::MongoStore.new(mongo: mongo, collection: collection, except: except)
     end
 
-    def self.s3_store(bucket_name = 'pages', aws_credential = {}, except = [])
-      require 'polipus/storage/s3_store'
-
-      if aws_credential[:access_key_id].nil? || aws_credential[:secret_access_key].nil?
-        fail 'You have to specify AWS crediantials: :access_key_id and :secret_access_key'
-      end
-
-      self::S3Store.new(
-        bucket: bucket_name,
-        access_key_id: aws_credential[:access_key_id],
-        secret_access_key: aws_credential[:secret_access_key],
-        except: except
-      )
+    def self.rethink_store(conn = nil, table = COLLECTION, except = [])
+      require 'polipus/storage/rethink_store'
+      conn ||= RethinkDB::RQL.new.connect(host: 'localhost', port: 28_015, db: 'polipus' )
+      fail "First argument must be a RethinkDB::Connection, got `#{conn.class}`" unless conn.is_a?(RethinkDB::Connection)
+      self::RethinkStore.new(conn: conn, table: table,  except: except)
     end
 
     def self.dev_null

data/lib/polipus/storage/mongo_store.rb

@@ -2,6 +2,7 @@
 require 'mongo'
 require 'zlib'
 require 'thread'
+
 module Polipus
   module Storage
     class MongoStore < Base
@@ -10,7 +11,11 @@ module Polipus
         @mongo      = options[:mongo]
         @collection = options[:collection]
         @mongo.create_collection(@collection)
-        @mongo[@collection].ensure_index(:uuid, unique: true, dropDups: true, background: true)
+        begin
+          @mongo[@collection].ensure_index(:uuid, unique: true, dropDups: true, background: true)
+        rescue Exception
+        end
+
         @compress_body = options[:compress_body] ||= true
         @except = options[:except] ||= []
         @semaphore = Mutex.new

data/lib/polipus/storage/rethink_store.rb

@@ -0,0 +1,90 @@
+# encoding: UTF-8
+require 'rethinkdb'
+require 'thread'
+require 'zlib'
+
+module Polipus
+  module Storage
+    class RethinkStore < Base
+      BINARY_FIELDS = %w(body headers data)
+      def initialize(options = {})
+        @r       = RethinkDB::RQL.new
+        @rethink = options[:conn]
+        @table   = options[:table]
+
+        unless @r.table_list.run(@rethink).include?(@table)
+          @r.table_create(@table).run(@rethink)
+          @r.table(@table).index_create('created_at')
+        end
+
+        @compress_body = options[:compress_body] ||= true
+        @except = options[:except] ||= []
+        @semaphore = Mutex.new
+      end
+
+      def add(page)
+        @semaphore.synchronize do
+          obj = page.to_hash
+          @except.each { |e| obj.delete e.to_s }
+          obj[:id] = uuid(page)
+          obj['body'] = Zlib::Deflate.deflate(obj['body']) if @compress_body && obj['body']
+          obj['created_at'] ||= Time.now.to_i
+          BINARY_FIELDS.each do |field|
+            # Use some marshalling?
+            obj[field] = @r.binary(obj[field]) unless obj[field].nil?
+          end
+
+          @r.table(@table).insert(obj).run(@rethink, durability: 'soft')
+          obj[:id]
+        end
+      end
+
+      def exists?(page)
+        @semaphore.synchronize do
+          doc = @r.table(@table).get(uuid(page)).run(@rethink)
+          !doc.nil?
+        end
+      end
+
+      def get(page)
+        @semaphore.synchronize do
+          data = @r.table(@table).get(uuid(page)).run(@rethink)
+          return load_page(data) if data
+        end
+      end
+
+      def remove(page)
+        @semaphore.synchronize do
+          @r.table(@table).get(uuid(page)).delete.run(@rethink)
+        end
+      end
+
+      def count
+        @r.table(@table).count.run(@rethink)
+      end
+
+      def each
+        @r.table(@table).run(@rethink).each do |doc|
+          page = load_page(doc)
+          yield doc[:id], page
+        end
+      end
+
+      def clear
+        @r.table(@table).delete.run(@rethink)
+      end
+
+      private
+
+      def load_page(hash)
+        BINARY_FIELDS.each do |field|
+          hash[field] = hash[field].to_s
+        end
+        hash['body'] = Zlib::Inflate.inflate(hash['body']) if @compress_body && hash['body'] && !hash['body'].empty?
+        page = Page.from_hash(hash)
+        page.fetched_at ||= hash['created_at']
+        page
+      end
+    end
+  end
+end