polipus 0.3.3 → 0.3.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MGZhM2Q1OWE5ZTkxZWE3Y2JhMDJkNzZkMWRjMWZjYjI4YmRhODJmYw==
+    MjkyMTAxZWU0ODJmMzI5OTcwZjI0ZTFlNzZjOTYxNzY1MGUxOTJjZQ==
   data.tar.gz: !binary |-
-    ODg4NTk5ZjhjNDM0ZjBhN2M2OWJlOWQyOTE4OWFiZmY1ZmQxMTZiYQ==
+    OTg4M2UzY2I2ODNkMWMxZDYwMzkxOGI5MmRhMWJkN2I0N2ViYTMyMg==
 SHA512:
   metadata.gz: !binary |-
-    NTdmYWZhMmMzYTQxOWY5OTBlNGE1ZWQzNWYwMThjMjAwNzQ4NmU4MDY4OWRi
-    OWVkNzEwNmYwNTY3OGI4NmFkZGJiNzNhN2I3M2ZjMDUxNjAzMDAyZjEyYmZj
-    ZGFkODliNDc5ZTYzYWE2ODNiMzdkMWFjZGExYjgzNTQyNmM4MDc=
+    YWE4ZjM0NDA1NWJiMGVkYzUyMTM5ZWJjN2I3YTU5YTI2NDIwNmI0ZTNkZTRl
+    NTg0MzQyOTgxMGYzZjBlZGVkMmE2ODJkZTA0ZTg3NzY0MTJjZTljYWEwN2Fm
+    YTc4MjI3NGViMmNlMjQzNWIxODVlMmNlNWJjNDFhMzE1MzQxMjk=
   data.tar.gz: !binary |-
-    NGYzNDc2ODQ3OTBiMzE5N2M5YmM3ZDliN2IyYWRmNjBiZDI3ZTYyYjkxNjBk
-    MDFhZjA0NDE5YjFkYzdiZTQxMjg0ZGEwODA3Nzk0YWZiNDRmY2I1MmE2ZWI4
-    MjQxNTU4NjgwOTM1ZDdlYjM4Mjg0NjhjY2M2OGU5MDA1YjNjMjY=
+    YjRlMmVjYzA4MDVlYzAxMmM0NGYwMWFiYWMwNmYxNDg3MmMyOTY4YzQzZDQ2
+    OWZjNWUzYjJlMzk3MDg5NjE4ZmQxYTk5MWQ1Y2M0ZmQ5MjhkYjQxZjdkNzRk
+    ZjZjNzkyMjJmM2M0MzNiNDNkOTQxOWYyMDc2MDdhZTJlNWE1Y2M=

data/lib/polipus.rb

@@ -106,9 +106,7 @@ module Polipus
 
       @storage      = @options[:storage] ||= Storage.dev_null
 
-      @http_pool    = []
       @workers_pool = []
-      @queues_pool  = []
 
       @follow_links_like  = []
       @skip_links_like    = []
@@ -150,8 +148,8 @@ module Polipus
       @options[:workers].times do |worker_number|
         @workers_pool << Thread.new do
           @logger.debug { "Start worker #{worker_number}" }
-          http  = @http_pool[worker_number]   ||= HTTP.new(@options)
-          queue = @queues_pool[worker_number] ||= queue_factory
+          http  =  HTTP.new(@options)
+          queue =  queue_factory
           queue.process(false, @options[:queue_timeout]) do |message|
 
             next if message.nil?
@@ -216,7 +214,7 @@ module Polipus
             if @options[:depth_limit] == false || @options[:depth_limit] > page.depth
               links_for(page).each do |url_to_visit|
                 next unless should_be_visited?(url_to_visit)
-                enqueue url_to_visit, page, queue
+                enqueue url_to_visit, page
               end
             else
               @logger.info { "[worker ##{worker_number}] Depth limit reached #{page.depth}" }
@@ -396,12 +394,12 @@ module Polipus
     end
 
     # The url is enqueued for a later visit
-    def enqueue(url_to_visit, current_page, queue)
+    def enqueue(url_to_visit, current_page)
       page_to_visit = Page.new(url_to_visit.to_s, referer: current_page.url.to_s, depth: current_page.depth + 1)
-      queue << page_to_visit.to_json
+      internal_queue << page_to_visit.to_json
       to_track = @options[:include_query_string_in_saved_page] ? url_to_visit.to_s : url_to_visit.to_s.gsub(/\?.*$/, '')
       url_tracker.visit to_track
-      @logger.debug { "Added [#{url_to_visit}] to the queue" }
+      @logger.debug { "Added (#{url_to_visit}) to the queue" }
     end
 
     # It creates a redis client
@@ -444,6 +442,7 @@ module Polipus
           removed, restored = @overflow_manager.perform
           @logger.info { "Overflow Manager: items removed=#{removed}, items restored=#{restored}, items stored=#{queue_overflow_adapter.size}" }
           sleep @options[:queue_overflow_manager_check_time]
+          break if SignalHandler.terminated?
         end
 
       end

data/lib/polipus/http.rb

@@ -209,8 +209,6 @@ module Polipus
     end
 
     def refresh_connection(url)
-      proxy_host, proxy_port = proxy_host_port unless @opts[:proxy_host_port].nil?
-
       if @opts[:logger] && proxy_host && proxy_port
         @opts[:logger].debug { "Request #{url} using proxy: #{proxy_host}:#{proxy_port}" }
       end

data/lib/polipus/version.rb

@@ -1,5 +1,5 @@
 # encoding: UTF-8
 module Polipus
-  VERSION = '0.3.3'
+  VERSION = '0.3.4'
   HOMEPAGE = 'https://github.com/taganaka/polipus'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polipus
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Francesco Laurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-27 00:00:00.000000000 Z
+date: 2014-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-bloomfilter