polipus 0.3.0 → 0.3.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZTc3MjQ1OWQwNzVhMWFhMGQ2NTdlYjM3ZTkyZDQ3ZDAwZDExZWQ1Mw==
+    MWJhN2NlNmRiZTcxODdhNGIzMWJmZWJhMDgwN2JhZTNhNjFlMGE2ZA==
   data.tar.gz: !binary |-
-    OTUzYTE5M2U4YTQ3ZGVmZTAzMzdiYjJmZWYzM2Q3MTU0NDMyYzAwMQ==
+    ZmE0NThkNzkwNDQ4MDQ5ZGRhZGViMzNmYzAwNWRlMzgyZDAwZmUyNg==
 SHA512:
   metadata.gz: !binary |-
-    MDk5MGQ2MzBkYzU2MjJlNDg1YTkwYTU1YjJjYWQ0YjAyNDY5OTZkNWJlZDIw
-    NDAwNjY2ZjMwMGUxZWE0NTNiNzc5YmIzZTg2NjcwNjFjZTMyNzIxZjZlYzZm
-    N2ZjMTk2ZjRkYjU0M2VjZDk0NWMxYzk0MjE4MWRkOWFiY2M3YTA=
+    YTNhOThhMzk4M2RhOGE4NzQ4NDM0OTBjOTlkNjYwNmI2YTlkZmU3MDNjMDQ2
+    NDlmZDVmZjQ0ZWJmZDFjYjJkYzFhYzJiMmYyYzRlOTc0N2RmY2NlMTU1ZDIy
+    YTZjNDU4NzZkYmQ3ZmI1ZjNjZTVmYTllOTE5OTkzNDI1ZjZjMzI=
   data.tar.gz: !binary |-
-    OTgwMTI5MWFhNWQ5Mjk4OWNmZTk3ZGE0MTMyYzM5NDlkMWJhMjFiMWQ4NDQ4
-    OGI1NDU3ZDQ0ZTkzNWFkMzAyZjg3YmRiNDlmN2I0ZDNlNWRlZmVkMjIzMWQ2
-    MGY0NGQ4YTQ1ZmEyMGQ0M2VkNzE2YzIyOGMxOGE4MDQzMWFkZjU=
+    MTUwYWRjY2VmZDk4Mzk5MWI5ZGNjMjFmZjViZWM2YjA2ZmZjZDViYTIzZGE5
+    OGRmY2U4MjNmZDBiNjBkMmNiZDZkNmM5MGNjYzNmODJlNDk0Nzk5OGFhNTdl
+    YjZjNzIyYzNjZjY1YzExNTU4YjBiYzAyM2VhYWI3YTY4NTA5N2M=

data/.rubocop.yml

@@ -0,0 +1,17 @@
+inherit_from: .rubocop_todo.yml
+AllCops:
+  Exclude:
+   - my_test/**/*
+   - examples/**/*
+
+Style/LineLength:
+  Enabled: false
+
+Style/TrivialAccessors:
+  Enabled: false
+
+Style/ClassLength:
+  Enabled: false
+
+Style/MethodLength:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,37 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2014-06-08 11:25:39 -0700 using RuboCop version 0.23.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Style/ClassVars:
+  Enabled: false
+
+# Offense count: 10
+Style/CyclomaticComplexity:
+  Max: 16
+
+# Offense count: 26
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 38
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/Encoding:
+  Enabled: false
+
+# Offense count: 2
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/Next:
+  Enabled: false
+
+# Offense count: 5
+# Configuration parameters: MaxSlashes.
+Style/RegexpLiteral:
+  Enabled: false
+
+# Offense count: 4
+Style/RescueModifier:
+  Enabled: false

data/.travis.yml

@@ -3,7 +3,8 @@ rvm:
   - jruby
   - 1.9.3
   - 2.0.0
-  - 2.1.1
+  - 2.1.2
+  - rbx-2
 
 services: 
   - mongodb

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.3.1 (2015-06-17)
+
+[Compare changes in gem](https://github.com/taganaka/polipus/compare/0.3.0...0.3.1)
+
+* Major Code-Style changes and cleanup
+  [#35](https://github.com/taganaka/polipus/pull/35)
+* BugFix: proper initialization of internal_queue
+  [#38](https://github.com/taganaka/polipus/pull/38)
+* Better INT / TERM Signal handling [#34](https://github.com/taganaka/polipus/pull/34)
+
+  New option added:
+    ```ruby
+    enable_signal_handler: true / false
+    ```
+  
+* Zlib::GzipFile::Error handling
+  [da3b927](https://github.com/taganaka/polipus/commit/da3b927acb1b50c26276ed458da0a365c22fd98b)
+* Faster and easier overflow management
+  [#39](https://github.com/taganaka/polipus/pull/39)
+
 ## 0.3.0 (2015-06-02)
 
 [Compare changes in gem](https://github.com/taganaka/polipus/compare/0.2.2...0.3.0)

data/README.md

@@ -21,6 +21,15 @@ http://www.slideshare.net/francescolaurita/roll-your-own-web-crawler-rubyday
 * Focus crawling made easy
 * Heavily inspired to Anemone https://github.com/chriskite/anemone/
 
+## Supported Ruby Interpreters
+
+* MRI 1.9.x >= 1.9.1
+* MRI 2.0.0
+* MRI 2.1.2
+* JRuby 1.9 mode
+* Rubinius
+
+
 ## Survival code example
 
 ```ruby
@@ -52,6 +61,7 @@ end
 * Start a feature/bugfix branch.
 * Commit and push until you are happy with your contribution.
 * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Install [Rubocop](https://github.com/bbatsov/rubocop) and make sure it is happy
 * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
 
 ## Copyright ##

data/Rakefile

@@ -1,9 +1,9 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec) do |spec|
   spec.pattern = 'spec/*_spec.rb'
 end
 
-task :default => :spec
-task :test    => :spec
+task default: :spec
+task test: :spec

data/examples/basic.rb

@@ -1,29 +1,29 @@
-require "polipus"
-require "mongo"
-require "polipus/plugins/cleaner"
+require 'polipus'
+require 'mongo'
+require 'polipus/plugins/cleaner'
 # Define a Mongo connection
-mongo = Mongo::Connection.new(:pool_size => 15, :pool_timeout => 5).db('crawler')
+mongo = Mongo::Connection.new(pool_size: 15, pool_timeout: 5).db('crawler')
 
 # Override some default options
 options = {
-  #Redis connection
-  :redis_options => {
-    :host => 'localhost',
-    :db   => 5, 
-    :driver => 'hiredis'
+  # Redis connection
+  redis_options: {
+    host: 'localhost',
+    db: 5,
+    driver: 'hiredis'
   },
   # Page storage: pages is the name of the collection where
   # pages will be stored
-  :storage    => Polipus::Storage.mongo_store(mongo, 'pages'),
+  storage: Polipus::Storage.mongo_store(mongo, 'pages'),
   # Use your custom user agent
-  :user_agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71",
+  user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71',
   # Use 5 threads
-  :workers    => 5,
+  workers: 5,
   # Logs goes to the crawler.log file
-  :logger     => Logger.new(STDOUT)
+  logger: Logger.new(STDOUT)
 }
-Polipus::Plugin.register Polipus::Plugin::Cleaner, reset:true
-starting_urls = ["http://rubygems.org/gems"]
+Polipus::Plugin.register Polipus::Plugin::Cleaner, reset: true
+starting_urls = ['http://rubygems.org/gems']
 
 # Crawl the entire rubygems's site
 # Polipus.crawler('polipus-rubygems', starting_urls, options)
@@ -49,10 +49,7 @@ Polipus.crawler('polipus-rubygems', starting_urls, options) do |crawler|
   end
 
   # Do a nifty stuff at the end of the crawling session
-  crawler.on_crawl_end do 
+  crawler.on_crawl_end do
     # Gong.bang(:loudly)
   end
 end
-
-
-

data/examples/incremental.rb

@@ -1,36 +1,36 @@
-require "polipus"
-require "mongo"
+require 'polipus'
+require 'mongo'
 
 # Define a Mongo connection
-mongo = Mongo::Connection.new(:pool_size => 15, :pool_timeout => 5).db('crawler')
+mongo = Mongo::Connection.new(pool_size: 15, pool_timeout: 5).db('crawler')
 # Override some default options
 options = {
-  #Redis connection
-  :redis_options => {
-    :host => 'localhost',
-    :db   => 5, 
-    :driver => 'hiredis'
+  # Redis connection
+  redis_options: {
+    host: 'localhost',
+    db: 5,
+    driver: 'hiredis'
   },
   # Page storage: pages is the name of the collection where
   # pages will be stored
-  :storage    => Polipus::Storage.mongo_store(mongo, 'pages'),
+  storage: Polipus::Storage.mongo_store(mongo, 'pages'),
   # Use your custom user agent
-  :user_agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71",
+  user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71',
   # Use 10 threads
-  :workers    => 20,
+  workers: 20,
   # Logs goes to the crawler.log file
-  :logger     => Logger.new(STDOUT),
+  logger: Logger.new(STDOUT),
   # Do not go deeper than 2 levels
-  :depth_limit   => 5,
+  depth_limit: 5,
 
   # Incremental download:
   # Set a ttl for each stored page
   # If a previous stored page is now expired, it will re-downloaded
   # Mark a page expired after 60s
-  :ttl_page => 60
+  ttl_page: 60
 }
 
-starting_urls = ["http://rubygems.org/gems"]
+starting_urls = ['http://rubygems.org/gems']
 
 # Crawl the entire rubygems's site
 # Polipus.crawler('polipus-rubygems', starting_urls, options)
@@ -52,11 +52,11 @@ Polipus.crawler('polipus-rubygems', starting_urls, options) do |crawler|
   # In-place page processing
   crawler.on_page_downloaded do |page|
     # A nokogiri object
-    puts "Page title: #{page.doc.css('title').text}" rescue "ERROR"
+    puts "Page title: #{page.doc.css('title').text}" rescue 'ERROR'
   end
 
   # Do a nifty stuff at the end of the crawling session
-  crawler.on_crawl_end do 
+  crawler.on_crawl_end do
     # Gong.bang(:loudly)
   end
 end

data/examples/robots_txt_handling.rb

@@ -6,7 +6,7 @@ options = {
 }
 
 Polipus.crawler('rubygems', 'http://rubygems.org/', options) do |crawler|
-  
+
   crawler.on_page_downloaded do |page|
     puts "Page title: '#{page.doc.at_css('title').content}' Page url: #{page.url}"
   end

data/examples/survival.rb

@@ -1,9 +1,9 @@
-require "polipus"
+require 'polipus'
 
-Polipus.crawler("rubygems","http://rubygems.org/") do |crawler|
+Polipus.crawler('rubygems', 'http://rubygems.org/') do |crawler|
   # In-place page processing
   crawler.on_page_downloaded do |page|
     # A nokogiri object
     puts "Page title: '#{page.doc.css('title').text}' Page url: #{page.url}"
   end
-end
+end

data/lib/polipus.rb

@@ -1,21 +1,20 @@
 # encoding: UTF-8
-require "redis"
-require "redis/connection/hiredis"
-require "redis-queue"
-require "polipus/version"
-require "polipus/http"
-require "polipus/storage"
-require "polipus/url_tracker"
-require "polipus/plugin"
-require "polipus/queue_overflow"
-require "polipus/robotex"
-require "thread"
-require "logger"
-require "json"
-require "singleton"
+require 'redis'
+require 'redis/connection/hiredis'
+require 'redis-queue'
+require 'polipus/version'
+require 'polipus/http'
+require 'polipus/storage'
+require 'polipus/url_tracker'
+require 'polipus/plugin'
+require 'polipus/queue_overflow'
+require 'polipus/robotex'
+require 'polipus/signal_handler'
+require 'thread'
+require 'logger'
+require 'json'
 
 module Polipus
-  
   def self.crawler(job_name = 'polipus', urls = [], options = {}, &block)
     PolipusCrawler.crawl(job_name, urls, options, &block)
   end
@@ -23,57 +22,61 @@ module Polipus
   class PolipusCrawler
     OPTS = {
       # run 4 threads
-      :workers => 4,
+      workers: 4,
       # identify self as Polipus/VERSION
-      :user_agent => "Polipus - #{Polipus::VERSION} - #{Polipus::HOMEPAGE}",
+      user_agent: "Polipus - #{Polipus::VERSION} - #{Polipus::HOMEPAGE}",
       # by default, don't limit the depth of the crawl
-      :depth_limit => false,
+      depth_limit: false,
       # number of times HTTP redirects will be followed
-      :redirect_limit => 5,
-      # storage engine defaults to DevNull 
-      :storage => nil,
-      # proxy server hostname 
-      :proxy_host => nil,
+      redirect_limit: 5,
+      # storage engine defaults to DevNull
+      storage: nil,
+      # proxy server hostname
+      proxy_host: nil,
       # proxy server port number
-      :proxy_port => false,
+      proxy_port: false,
       # HTTP read timeout in seconds
-      :read_timeout => 30,
+      read_timeout: 30,
       # HTTP open connection timeout in seconds
-      :open_timeout => 10,
+      open_timeout: 10,
       # Time to wait for new messages on Redis
       # After this timeout, current crawling session is marked as terminated
-      :queue_timeout => 30,
+      queue_timeout: 30,
       # An URL tracker instance. default is Bloomfilter based on redis
-      :url_tracker => nil,
+      url_tracker: nil,
       # A Redis options {} that will be passed directly to Redis.new
-      :redis_options => {},
+      redis_options: {},
       # An instance of logger
-      :logger => nil,
+      logger: nil,
       # A logger level
-      :logger_level => nil,
+      logger_level: nil,
       # whether the query string should be included in the saved page
-      :include_query_string_in_saved_page => true,
+      include_query_string_in_saved_page: true,
       # Max number of items to keep on redis
-      :queue_items_limit => 2_000_000,
+      queue_items_limit: 2_000_000,
       # The adapter used to store exceed (queue_items_limit) redis items
-      :queue_overflow_adapter => nil,
+      queue_overflow_adapter: nil,
       # Every x seconds, the main queue is checked for overflowed items
-      :queue_overflow_manager_check_time => 60,
+      queue_overflow_manager_check_time: 60,
       # If true, each page downloaded will increment a counter on redis
-      :stats_enabled => false,
+      stats_enabled: false,
       # Cookies strategy
-      :cookie_jar => nil,
-      # whether or not accept cookies 
-      :accept_cookies => false,
+      cookie_jar: nil,
+      # whether or not accept cookies
+      accept_cookies: false,
       # A set of hosts that should be considered parts of the same domain
       # Eg It can be used to follow links with and without 'www' domain
-      :domain_aliases => [],
+      domain_aliases: [],
       # Mark a connection as staled after connection_max_hits request
-      :connection_max_hits => nil,
+      connection_max_hits: nil,
       # Page TTL: mark a page as expired after ttl_page seconds
-      :ttl_page => nil,
+      ttl_page: nil,
       # don't obey the robots exclusion protocol
-      :obey_robots_txt => false
+      obey_robots_txt: false,
+      # If true, signal handling strategy is enabled.
+      # INT and TERM signal will stop polipus gracefully
+      # Disable it if polipus will run as a part of Resque or DelayedJob-like system
+      enable_signal_handler: true
     }
 
     attr_reader :storage
@@ -82,7 +85,6 @@ module Polipus
     attr_reader :options
     attr_reader :crawler_name
 
-
     OPTS.keys.each do |key|
       define_method "#{key}=" do |value|
         @options[key.to_sym] = value
@@ -93,13 +95,12 @@ module Polipus
     end
 
     def initialize(job_name = 'polipus', urls = [], options = {})
-
       @job_name     = job_name
       @options      = OPTS.merge(options)
       @options[:queue_timeout] = 1 if @options[:queue_timeout] <= 0
       @logger       = @options[:logger]  ||= Logger.new(nil)
-      
-      unless @logger.class.to_s == "Log4r::Logger"
+
+      unless @logger.class.to_s == 'Log4r::Logger'
         @logger.level = @options[:logger_level] ||= Logger::INFO
       end
 
@@ -108,8 +109,7 @@ module Polipus
       @http_pool    = []
       @workers_pool = []
       @queues_pool  = []
-      
-      
+
       @follow_links_like  = []
       @skip_links_like    = []
       @on_page_downloaded = []
@@ -119,21 +119,19 @@ module Polipus
       @on_crawl_end       = []
       @redis_factory      = nil
 
-      
       @overflow_manager = nil
       @crawler_name = `hostname`.strip + "-#{@job_name}"
 
       @storage.include_query_string_in_uuid = @options[:include_query_string_in_saved_page]
 
-      @urls = [urls].flatten.map{ |url| URI(url) }
-      @urls.each{ |url| url.path = '/' if url.path.empty? }
-      @internal_queue = queue_factory
+      @urls = [urls].flatten.map { |url| URI(url) }
+      @urls.each { |url| url.path = '/' if url.path.empty? }
       @robots = Polipus::Robotex.new(@options[:user_agent]) if @options[:obey_robots_txt]
-
+      # Attach signal handling if enabled
+      SignalHandler.enable if @options[:enable_signal_handler]
       execute_plugin 'on_initialize'
 
       yield self if block_given?
-
     end
 
     def self.crawl(*args, &block)
@@ -141,18 +139,17 @@ module Polipus
     end
 
     def takeover
-      PolipusSignalHandler.enable
       overflow_items_controller if queue_overflow_adapter
 
       @urls.each do |u|
         add_url(u) { |page| page.user_data.p_seeded = true }
       end
-      return if @internal_queue.empty?
+      return if internal_queue.empty?
 
       execute_plugin 'on_crawl_start'
       @options[:workers].times do |worker_number|
         @workers_pool << Thread.new do
-          @logger.debug {"Start worker #{worker_number}"}
+          @logger.debug { "Start worker #{worker_number}" }
           http  = @http_pool[worker_number]   ||= HTTP.new(@options)
           queue = @queues_pool[worker_number] ||= queue_factory
           queue.process(false, @options[:queue_timeout]) do |message|
@@ -164,75 +161,73 @@ module Polipus
             page = Page.from_json message
 
             unless should_be_visited?(page.url, false)
-              @logger.info {"[worker ##{worker_number}] Page (#{page.url.to_s}) is no more welcome."}
+              @logger.info { "[worker ##{worker_number}] Page (#{page.url}) is no more welcome." }
               queue.commit
               next
             end
 
             if page_exists? page
-              @logger.info {"[worker ##{worker_number}] Page (#{page.url.to_s}) already stored."}
+              @logger.info { "[worker ##{worker_number}] Page (#{page.url}) already stored." }
               queue.commit
               next
             end
-            
+
             url = page.url.to_s
-            @logger.debug {"[worker ##{worker_number}] Fetching page: [#{page.url.to_s}] Referer: #{page.referer} Depth: #{page.depth}"}
+            @logger.debug { "[worker ##{worker_number}] Fetching page: [#{page.url}] Referer: #{page.referer} Depth: #{page.depth}" }
 
             execute_plugin 'on_before_download'
 
             pages = http.fetch_pages(url, page.referer, page.depth)
             if pages.count > 1
               rurls = pages.map { |e| e.url.to_s }.join(' --> ')
-              @logger.info {"Got redirects! #{rurls}"}
+              @logger.info { "Got redirects! #{rurls}" }
               page = pages.pop
-              page.aliases = pages.collect { |e| e.url }
+              page.aliases = pages.map { |e| e.url }
               if page_exists? page
-                @logger.info {"[worker ##{worker_number}] Page (#{page.url.to_s}) already stored."}
+                @logger.info { "[worker ##{worker_number}] Page (#{page.url}) already stored." }
                 queue.commit
                 next
               end
             else
               page = pages.last
             end
-            
+
             execute_plugin 'on_after_download'
-            
+
             if page.error
-              @logger.warn {"Page #{page.url} has error: #{page.error}"}
+              @logger.warn { "Page #{page.url} has error: #{page.error}" }
               incr_error
-              @on_page_error.each {|e| e.call(page)}
+              @on_page_error.each { |e| e.call(page) }
             end
 
             # Execute on_before_save blocks
-            @on_before_save.each {|e| e.call(page)}
+            @on_before_save.each { |e| e.call(page) }
 
-            if page.storable?
-              @storage.add page
-            end
-            
-            @logger.debug {"[worker ##{worker_number}] Fetched page: [#{page.url.to_s}] Referrer: [#{page.referer}] Depth: [#{page.depth}] Code: [#{page.code}] Response Time: [#{page.response_time}]"}
-            @logger.info  {"[worker ##{worker_number}] Page (#{page.url.to_s}) downloaded"}
+            page.storable? && @storage.add(page)
+
+            @logger.debug { "[worker ##{worker_number}] Fetched page: [#{page.url}] Referrer: [#{page.referer}] Depth: [#{page.depth}] Code: [#{page.code}] Response Time: [#{page.response_time}]" }
+            @logger.info  { "[worker ##{worker_number}] Page (#{page.url}) downloaded" }
 
             incr_pages
 
             # Execute on_page_downloaded blocks
-            @on_page_downloaded.each {|e| e.call(page)}
+            @on_page_downloaded.each { |e| e.call(page) }
 
-            if @options[:depth_limit] == false || @options[:depth_limit] > page.depth 
+            if @options[:depth_limit] == false || @options[:depth_limit] > page.depth
               links_for(page).each do |url_to_visit|
                 next unless should_be_visited?(url_to_visit)
                 enqueue url_to_visit, page, queue
               end
             else
-              @logger.info {"[worker ##{worker_number}] Depth limit reached #{page.depth}"}
+              @logger.info { "[worker ##{worker_number}] Depth limit reached #{page.depth}" }
             end
 
-            @logger.debug {"[worker ##{worker_number}] Queue size: #{queue.size}"}
+            @logger.debug { "[worker ##{worker_number}] Queue size: #{queue.size}" }
             @overflow_manager.perform if @overflow_manager && queue.empty?
             execute_plugin 'on_message_processed'
 
-            if PolipusSignalHandler.terminated?
-              @logger.info {"About to exit! Thanks for using Polipus"}
+            if SignalHandler.terminated?
+              @logger.info { 'About to exit! Thanks for using Polipus' }
               queue.commit
               break
             end
@@ -240,11 +235,11 @@ module Polipus
           end
         end
       end
-      @workers_pool.each {|w| w.join}
-      @on_crawl_end.each {|e| e.call(self)}
+      @workers_pool.each { |w| w.join }
+      @on_crawl_end.each { |e| e.call(self) }
       execute_plugin 'on_crawl_end'
     end
-    
+
     # A pattern or an array of patterns can be passed as argument
     # An url will be discarded if it doesn't match patterns
     def follow_links_like(*patterns)
@@ -298,11 +293,11 @@ module Polipus
     end
 
     def queue_size
-      @internal_queue.size
+      internal_queue.size
     end
 
     def stats_reset!
-      ["polipus:#{@job_name}:errors", "polipus:#{@job_name}:pages"].each {|e| redis.del e}
+      ["polipus:#{@job_name}:errors", "polipus:#{@job_name}:pages"].each { |e| redis.del e }
     end
 
     def redis_factory(&block)
@@ -313,9 +308,9 @@ module Polipus
     def url_tracker
       @url_tracker ||=
         @options[:url_tracker] ||=
-          UrlTracker.bloomfilter(:key_name => "polipus_bf_#{job_name}",
-                                 :redis => redis_factory_adapter,
-                                 :driver => 'lua')
+          UrlTracker.bloomfilter(key_name: "polipus_bf_#{job_name}",
+                                 redis: redis_factory_adapter,
+                                 driver: 'lua')
     end
 
     def redis
@@ -334,176 +329,138 @@ module Polipus
     def add_url(url, params = {})
       page = Page.new(url, params)
       yield(page) if block_given?
-      @internal_queue << page.to_json
+      internal_queue << page.to_json
     end
 
     # Request to Polipus to stop its work (gracefully)
     # cler_queue = true if you want to delete all of the pending urls to visit
     def stop!(cler_queue = false)
-      PolipusSignalHandler.terminate
-      @internal_queue.clear(true) if cler_queue
+      SignalHandler.terminate
+      internal_queue.clear(true) if cler_queue
     end
 
     private
-      # URLs enqueue policy
-      def should_be_visited?(url, with_tracker = true)
-        
-        case
-        # robots.txt
-        when !allowed_by_robot?(url)
-          false
-        # Check against whitelist pattern matching
-        when !@follow_links_like.empty? && @follow_links_like.none?{ |p| url.path =~ p }
-          false
-        # Check against blacklist pattern matching
-        when @skip_links_like.any?{ |p| url.path =~ p }
-          false
-        # Page is marked as expired
-        when page_expired?(Page.new(url))
-          true
-        # Check against url tracker
-        when with_tracker && url_tracker.visited?(@options[:include_query_string_in_saved_page] ? url.to_s : url.to_s.gsub(/\?.*$/,''))
-          false
-        else
-          true
-        end
-      end
-
-      # It extracts URLs from the page
-      def links_for page
-        page.domain_aliases = domain_aliases
-        @focus_crawl_block.nil? ? page.links : @focus_crawl_block.call(page)
-      end
-
-      # whether a page is expired or not
-      def page_expired? page
-        return false if @options[:ttl_page].nil?
-        stored_page = @storage.get(page)
-        r = stored_page && stored_page.expired?(@options[:ttl_page])
-        @logger.debug {"Page #{page.url.to_s} marked as expired"} if r
-        r
-      end
-
-      # whether a page exists or not
-      def page_exists? page
-        return false if page.user_data && page.user_data.p_seeded
-        @storage.exists?(page) && !page_expired?(page)
-      end
 
-      #
-      # Returns +true+ if we are obeying robots.txt and the link
-      # is granted access in it. Always returns +true+ when we are
-      # not obeying robots.txt.
-      #
-      def allowed_by_robot?(link)
-        return true if @robots.nil?
-        @options[:obey_robots_txt] ? @robots.allowed?(link) : true
+    # URLs enqueue policy
+    def should_be_visited?(url, with_tracker = true)
+      case
+      # robots.txt
+      when !allowed_by_robot?(url)
+        false
+      # Check against whitelist pattern matching
+      when !@follow_links_like.empty? && @follow_links_like.none? { |p| url.path =~ p }
+        false
+      # Check against blacklist pattern matching
+      when @skip_links_like.any? { |p| url.path =~ p }
+        false
+      # Page is marked as expired
+      when page_expired?(Page.new(url))
+        true
+      # Check against url tracker
+      when with_tracker && url_tracker.visited?(@options[:include_query_string_in_saved_page] ? url.to_s : url.to_s.gsub(/\?.*$/, ''))
+        false
+      else
+        true
       end
+    end
 
+    # It extracts URLs from the page
+    def links_for(page)
+      page.domain_aliases = domain_aliases
+      @focus_crawl_block.nil? ? page.links : @focus_crawl_block.call(page)
+    end
 
-      # The url is enqueued for a later visit
-      def enqueue url_to_visit, current_page, queue
-        page_to_visit = Page.new(url_to_visit.to_s, :referer => current_page.url.to_s, :depth => current_page.depth + 1)
-        queue << page_to_visit.to_json
-        to_track = @options[:include_query_string_in_saved_page] ? url_to_visit.to_s : url_to_visit.to_s.gsub(/\?.*$/,'')
-        url_tracker.visit to_track
-        @logger.debug {"Added [#{url_to_visit.to_s}] to the queue"}        
-      end
+    # whether a page is expired or not
+    def page_expired?(page)
+      return false if @options[:ttl_page].nil?
+      stored_page = @storage.get(page)
+      r = stored_page && stored_page.expired?(@options[:ttl_page])
+      @logger.debug { "Page #{page.url} marked as expired" } if r
+      r
+    end
 
-      # It creates a redis client
-      def redis_factory_adapter
-        if @redis_factory
-          @redis_factory.call(redis_options)
-        else
-          Redis.new(redis_options)
-        end
-      end
+    # whether a page exists or not
+    def page_exists?(page)
+      return false if page.user_data && page.user_data.p_seeded
+      @storage.exists?(page) && !page_expired?(page)
+    end
 
-      # It creates a new distributed queue
-      def queue_factory
-        Redis::Queue.new("polipus_queue_#{@job_name}","bp_polipus_queue_#{@job_name}", :redis => redis_factory_adapter)
-      end
+    #
+    # Returns +true+ if we are obeying robots.txt and the link
+    # is granted access in it. Always returns +true+ when we are
+    # not obeying robots.txt.
+    #
+    def allowed_by_robot?(link)
+      return true if @robots.nil?
+      @options[:obey_robots_txt] ? @robots.allowed?(link) : true
+    end
 
-      # If stats enable, it increments errors found
-      def incr_error
-        redis.incr "polipus:#{@job_name}:errors" if @options[:stats_enabled]
-      end
+    # The url is enqueued for a later visit
+    def enqueue(url_to_visit, current_page, queue)
+      page_to_visit = Page.new(url_to_visit.to_s, referer: current_page.url.to_s, depth: current_page.depth + 1)
+      queue << page_to_visit.to_json
+      to_track = @options[:include_query_string_in_saved_page] ? url_to_visit.to_s : url_to_visit.to_s.gsub(/\?.*$/, '')
+      url_tracker.visit to_track
+      @logger.debug { "Added [#{url_to_visit}] to the queue" }
+    end
 
-      # If stats enable, it increments pages downloaded
-      def incr_pages
-        redis.incr "polipus:#{@job_name}:pages" if @options[:stats_enabled]
+    # It creates a redis client
+    def redis_factory_adapter
+      if @redis_factory
+        @redis_factory.call(redis_options)
+      else
+        Redis.new(redis_options)
       end
+    end
 
-      # It handles the overflow item policy (if any)
-      def overflow_items_controller
-        @overflow_manager = QueueOverflow::Manager.new(self, queue_factory, @options[:queue_items_limit])
-
-        # In the time, url policy may change so policy is re-evaluated
-        @overflow_manager.url_filter do |page|
-          should_be_visited?(page.url, false)
-        end
+    # It creates a new distributed queue
+    def queue_factory
+      Redis::Queue.new("polipus_queue_#{@job_name}", "bp_polipus_queue_#{@job_name}", redis: redis_factory_adapter)
+    end
 
-        Thread.new do
-         
-          redis_lock = redis_factory_adapter
-          op_timeout = @options[:queue_overflow_manager_check_time]
+    # If stats enabled, it increments errors found
+    def incr_error
+      redis.incr "polipus:#{@job_name}:errors" if @options[:stats_enabled]
+    end
 
-          while true
-            lock = redis_lock.setnx "polipus_queue_overflow-#{@job_name}.lock", 1
+    # If stats enabled, it increments pages downloaded
+    def incr_pages
+      redis.incr "polipus:#{@job_name}:pages" if @options[:stats_enabled]
+    end
 
-            if lock
-              redis_lock.expire "polipus_queue_overflow-#{@job_name}.lock", op_timeout + 350
-              removed, restored = @overflow_manager.perform
-              @logger.info {"Overflow Manager: items removed=#{removed}, items restored=#{restored}, items stored=#{queue_overflow_adapter.size}"}
-              redis_lock.del "polipus_queue_overflow-#{@job_name}.lock"
-            else
-              @logger.info {"Lock not acquired"}
-            end
+    # It handles the overflow item policy (if any)
+    def overflow_items_controller
+      @overflow_manager = QueueOverflow::Manager.new(self, queue_factory, @options[:queue_items_limit])
 
-            sleep @options[:queue_overflow_manager_check_time]
-          end
-        end
+      # In the time, url policy may change so policy is re-evaluated
+      @overflow_manager.url_filter do |page|
+        should_be_visited?(page.url, false)
       end
 
-      # It invokes a plugin method if any
-      def execute_plugin method
+      Thread.new do
 
-        Polipus::Plugin.plugins.each do |k,p|
-          if p.respond_to? method
-            @logger.info("Running plugin method #{method} on #{k}")
-            ret_val = p.send(method, self)
-            instance_eval(&ret_val) if ret_val.kind_of? Proc
-          end
+        loop do
+          @logger.info { 'Overflow Manager: cycle started' }
+          removed, restored = @overflow_manager.perform
+          @logger.info { "Overflow Manager: items removed=#{removed}, items restored=#{restored}, items stored=#{queue_overflow_adapter.size}" }
+          sleep @options[:queue_overflow_manager_check_time]
         end
-      end
 
-  end
-
-  class PolipusSignalHandler
-    include Singleton
-    attr_accessor :terminated
-    def initialize
-      self.terminated = false
-    end
-
-    def self.enable
-      trap(:INT)  {
-        puts "Got INT signal"
-        self.terminate
-      }
-      trap(:TERM) {
-        puts "Got TERM signal"
-        self.terminate
-      }
+      end
     end
 
-    def self.terminate
-      self.instance.terminated = true
+    def internal_queue
+      @internal_queue ||= queue_factory
     end
 
-    def self.terminated?
-      self.instance.terminated
+    # It invokes a plugin method if any
+    def execute_plugin(method)
+      Polipus::Plugin.plugins.each do |k, p|
+        next unless p.respond_to?(method)
+        @logger.info { "Running plugin method #{method} on #{k}" }
+        ret_val = p.send(method, self)
+        instance_eval(&ret_val) if ret_val.kind_of? Proc
+      end
     end
   end
-
 end