polipus 0.2.1 → 0.2.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MDlmZGJiYmY1MWQ1NWFiMDMxZGRmYzBmNGQ4OGU4ZGFmOTgxN2M5Mg==
+    Mzc5NDczMjAxNmUzY2JlYzA4YzUwZDUzMGY0Y2ZjZGQxOTBmMTk5YQ==
   data.tar.gz: !binary |-
-    Y2NjYmI5N2Y1NDA4NjVlNGQ2YWNhMWUxYjkyODA5NjBkOWVmOTljZQ==
+    NDgxNzhlYTAyNTMxYWE1MTBkMzIxMWMyMDRlMmQ0NjA2ZTc1MzY2Mw==
 SHA512:
   metadata.gz: !binary |-
-    MmQ1ZDc2M2RiYmVmZjg0NDVlYjYyNTYzOWY4M2NjMGQ1MDZjMGQwZWYwNjJj
-    OWMzNjY4ODI3ZTA5ZmEzM2U0YTZiNWQxOWI2YzkyNDAzM2Q5YzIyZDFiNmE5
-    MzQ0YzM3ZDgwNTk5N2ZjNzMxOTY3OTllNWFjNDI3YjQ0NGVhODg=
+    YTNmMGY4YWRkNmU5YThhMTY0ODBiMzY1MmI3MzkzMzEzZjhiZGI4NDI1Mjdh
+    MGI0YWNjZDEyM2NhZDlkZTY2MjY3ZTA5YzVjNWE5YjFjZWEwZjRhMGI3ZWE2
+    MTc0NDczOWIyYTE1YjY1MzA2ZjcwOTRjNWZiOGVlMjIyOTJkNGU=
   data.tar.gz: !binary |-
-    ZTI1NWUzNzQwNTA2NTU4Mjk5OTVhNmI3NDlkNDFmOTRmN2JmZDhhZDRhYWZl
-    Y2VkM2ZkNDZjZWYyOTk0NTgzZTIxOTA0ZGEzMjc1MGNmOGFkYTgxZjc4ZWJj
-    ZGVhY2FiNDk3MDYwNzllMGNiYTI2NDMwYTAyMzY4NTczYmJjNGQ=
+    ZTVmZGQ0NTgyNTA2NGYwZDVhMTg3NmUzMWM0NDExMGIwOTU5NjM2ZGZkNWM3
+    MWYyYWQwM2NkMmNiMDgxYWY3NTdmYjE5NDZhOGE1YTQxNjRjNjUxNTIzYTc2
+    NTU2ZDZiYzllNWYxZjdiNDIwMzYyNGYyN2YyM2ZmY2VmNWU1NzA=

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+rvm:
+  - jruby
+  - 1.9.3
+  - 2.0.0
+  - 2.1.1
+
+services: 
+  - mongodb
+  - redis
+

data/AUTHORS

@@ -0,0 +1,2 @@
+Francesco Laurita <francesco.laurita@gmail.com>
+Tobias L. Maier <http://tobiasmaier.info/>

data/README.md

@@ -1,3 +1,9 @@
+[![Build Status](https://travis-ci.org/taganaka/polipus.svg?branch=master)](https://travis-ci.org/taganaka/polipus)
+[![Coverage Status](https://coveralls.io/repos/taganaka/polipus/badge.png?branch=master)](https://coveralls.io/r/taganaka/polipus?branch=master)
+[![Code Climate](https://codeclimate.com/github/taganaka/polipus.png)](https://codeclimate.com/github/taganaka/polipus)
+[![RubyGems](http://img.shields.io/gem/v/polipus.svg)](https://rubygems.org/gems/polipus)
+
+
 # Polipus #
 
 A distributed web crawler written in ruby, backed by Redis

data/lib/polipus.rb

@@ -15,7 +15,7 @@ require "singleton"
 
 module Polipus
   
-  def Polipus.crawler(job_name = 'polipus', urls = [], options = {}, &block)
+  def self.crawler(job_name = 'polipus', urls = [], options = {}, &block)
     PolipusCrawler.crawl(job_name, urls, options, &block)
   end
 
@@ -75,7 +75,6 @@ module Polipus
     attr_reader :storage
     attr_reader :job_name
     attr_reader :logger
-    attr_reader :overflow_adapter
     attr_reader :options
     attr_reader :crawler_name
 
@@ -89,7 +88,7 @@ module Polipus
       end
     end
 
-    def initialize(job_name = 'polipus',urls = [], options = {})
+    def initialize(job_name = 'polipus', urls = [], options = {})
 
       @job_name     = job_name
       @options      = OPTS.merge(options)
@@ -121,22 +120,19 @@ module Polipus
 
       @storage.include_query_string_in_uuid = @options[:include_query_string_in_saved_page]
 
-      @urls = [urls].flatten.map{ |url| url.is_a?(URI) ? url : URI(url) }
+      @urls = [urls].flatten.map{ |url| URI(url) }
       @urls.each{ |url| url.path = '/' if url.path.empty? }
+
+      @internal_queue = queue_factory
+
       execute_plugin 'on_initialize'
 
       yield self if block_given?
 
     end
 
-    def self.crawl(job_name, urls, opts = {})
-
-      self.new(job_name, urls, opts) do |polipus|
-        yield polipus if block_given?
-        
-        polipus.takeover
-      end
-      
+    def self.crawl(*args, &block)
+      new(*args, &block).takeover
     end
 
     def takeover
@@ -167,13 +163,13 @@ module Polipus
             page = Page.from_json message
 
             unless should_be_visited?(page.url, false)
-              @logger.info {"[worker ##{worker_number}] Page [#{page.url.to_s}] is no more welcome."}
+              @logger.info {"[worker ##{worker_number}] Page (#{page.url.to_s}) is no more welcome."}
               queue.commit
               next
             end
 
             if page_exists? page
-              @logger.info {"[worker ##{worker_number}] Page [#{page.url.to_s}] already stored."}
+              @logger.info {"[worker ##{worker_number}] Page (#{page.url.to_s}) already stored."}
               queue.commit
               next
             end
@@ -190,7 +186,7 @@ module Polipus
               page = pages.pop
               page.aliases = pages.collect { |e| e.url }
               if page_exists? page
-                @logger.info {"[worker ##{worker_number}] Page [#{page.url.to_s}] already stored."}
+                @logger.info {"[worker ##{worker_number}] Page (#{page.url.to_s}) already stored."}
                 queue.commit
                 next
               end
@@ -212,7 +208,7 @@ module Polipus
             
             if page
               @logger.debug {"[worker ##{worker_number}] Fetched page: [#{page.url.to_s}] Referrer: [#{page.referer}] Depth: [#{page.depth}] Code: [#{page.code}] Response Time: [#{page.response_time}]"}
-              @logger.info  {"[worker ##{worker_number}] Page [#{page.url.to_s}] downloaded"}
+              @logger.info  {"[worker ##{worker_number}] Page (#{page.url.to_s}) downloaded"}
             end
             
             incr_pages
@@ -261,7 +257,7 @@ module Polipus
       self
     end
 
-    # A block of code will be executed on every page dowloaded
+    # A block of code will be executed on every page downloaded
     # The block takes the page as argument
     def on_page_downloaded(&block)
       @on_page_downloaded << block
@@ -292,12 +288,7 @@ module Polipus
       @options[:redis_options]
     end
 
-    def overflow_adapter
-      @options[:overflow_adapter]
-    end
-
     def queue_size
-      @internal_queue ||= queue_factory
       @internal_queue.size
     end
 
@@ -311,63 +302,58 @@ module Polipus
     end
 
     def url_tracker
-      if @url_tracker.nil?
-        @url_tracker  = @options[:url_tracker] ||= UrlTracker.bloomfilter(:key_name => "polipus_bf_#{job_name}", :redis => redis_factory_adapter, :driver => 'lua')
-      end
-      @url_tracker
+      @url_tracker ||=
+        @options[:url_tracker] ||=
+          UrlTracker.bloomfilter(:key_name => "polipus_bf_#{job_name}",
+                                 :redis => redis_factory_adapter,
+                                 :driver => 'lua')
     end
 
     def redis
-      if @redis.nil?
-        @redis = redis_factory_adapter
-      end
-      @redis
+      @redis ||= redis_factory_adapter
     end
 
+    # Enqueue an url, no matter what
     def add_url url
-      @url_tracker.remove url.to_s
       page = Page.new(url)
-      queue_factory << page.to_json
+      @internal_queue << page.to_json
     end
 
     # Request to Polipus to stop its work (gracefully)
     # cler_queue = true if you want to delete all of the pending urls to visit
     def stop!(cler_queue = false)
       PolipusSignalHandler.terminate
-      queue_factory.clear(true) if cler_queue
+      @internal_queue.clear(true) if cler_queue
     end
 
     private
       # URLs enqueue policy
       def should_be_visited?(url, with_tracker = true)
-
+        case
         # Check against whitelist pattern matching
-        unless @follow_links_like.empty?
-          return false unless @follow_links_like.any?{|p| url.path =~ p}  
-        end
-
+        when !@follow_links_like.empty? && @follow_links_like.none?{ |p| url.path =~ p }
+          false
         # Check against blacklist pattern matching
-        unless @skip_links_like.empty?
-          return false if @skip_links_like.any?{|p| url.path =~ p}
-        end
-
-        #Page is marked as expired
-        return true if page_expired?(Page.new(url))
-
+        when @skip_links_like.any?{ |p| url.path =~ p }
+          false
+        # Page is marked as expired
+        when page_expired?(Page.new(url))
+          true
         # Check against url tracker
-        if with_tracker
-          return false if url_tracker.visited?(@options[:include_query_string_in_saved_page] ? url.to_s : url.to_s.gsub(/\?.*$/,''))
+        when with_tracker && url_tracker.visited?(@options[:include_query_string_in_saved_page] ? url.to_s : url.to_s.gsub(/\?.*$/,''))
+          false
+        else
+          true
         end
-        true
       end
 
       # It extracts URLs from the page
       def links_for page
         page.domain_aliases = domain_aliases
-        links = @focus_crawl_block.nil? ? page.links : @focus_crawl_block.call(page)
-        links
+        @focus_crawl_block.nil? ? page.links : @focus_crawl_block.call(page)
       end
 
+      # whether a page is expired or not
       def page_expired? page
         return false if @options[:ttl_page].nil?
         stored_page = @storage.get(page)
@@ -376,6 +362,7 @@ module Polipus
         r
       end
 
+      # whether a page exists or not
       def page_exists? page
         return false if page.user_data && page.user_data.p_seeded
         @storage.exists?(page) && !page_expired?(page)
@@ -392,10 +379,11 @@ module Polipus
 
       # It creates a redis client
       def redis_factory_adapter
-        unless @redis_factory.nil?
-          return @redis_factory.call(redis_options)
+        if @redis_factory
+          @redis_factory.call(redis_options)
+        else
+          Redis.new(redis_options)
         end
-        Redis.new(redis_options)
       end
 
       # It creates a new distributed queue

data/lib/polipus/http.rb

@@ -27,33 +27,33 @@ module Polipus
     # including redirects
     #
     def fetch_pages(url, referer = nil, depth = nil)
-      begin
-        url = URI(url) unless url.is_a?(URI)
-        pages = []
-        get(url, referer) do |response, code, location, redirect_to, response_time|
-          body = response.body.dup
-          if response.to_hash.fetch('content-encoding', [])[0] == 'gzip'
-            gzip = Zlib::GzipReader.new(StringIO.new(body))    
-            body = gzip.read
-          end
-          pages << Page.new(location, :body          => response.body.dup,
-                                      :code          => code,
-                                      :headers       => response.to_hash,
-                                      :referer       => referer,
-                                      :depth         => depth,
-                                      :redirect_to   => redirect_to,
-                                      :response_time => response_time,
-                                      :fetched_at    => Time.now.to_i)
+      url = URI(url)
+      pages = []
+      get(url, referer) do |response, code, location, redirect_to, response_time|
+        body = response.body.dup
+        if response.to_hash.fetch('content-encoding', [])[0] == 'gzip'
+          gzip = Zlib::GzipReader.new(StringIO.new(body))
+          body = gzip.read
         end
+      
+        pages << Page.new(location, :body          => body,
+                                    :code          => code,
+                                    :headers       => response.to_hash,
+                                    :referer       => referer,
+                                    :depth         => depth,
+                                    :redirect_to   => redirect_to,
+                                    :response_time => response_time,
+                                    :fetched_at    => Time.now.to_i)
+      end
 
-        return pages
-      rescue StandardError => e
-        if verbose?
-          puts e.inspect
-          puts e.backtrace
-        end
-        return [Page.new(url, :error => e)]
+      pages
+    rescue StandardError => e
+      if verbose?
+        puts e.inspect
+        puts e.backtrace
       end
+
+      [Page.new(url, :error => e)]
     end
 
     #
@@ -154,6 +154,8 @@ module Polipus
       opts['User-Agent'] = user_agent if user_agent
       opts['Referer'] = referer.to_s if referer
       opts['Cookie']  = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
+      opts['Accept-Encoding'] = 'gzip'
+
 
       retries = 0
       begin
@@ -227,5 +229,9 @@ module Polipus
       to_url.host.nil? || (to_url.host == from_url.host)
     end
 
+    def gzip_enabled?
+      @opts[:gzip_enabled]
+    end
+
   end
 end

data/lib/polipus/version.rb

@@ -1,4 +1,4 @@
 module Polipus
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
   HOMEPAGE = "https://github.com/taganaka/polipus"
 end

data/polipus.gemspec

@@ -40,6 +40,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'vcr', '~> 2.5', '>= 2.5.0'
   s.add_development_dependency 'webmock', '>= 1.8.0', '< 1.12'
   s.add_development_dependency 'flexmock', '~> 1.3', '>= 1.3.2'
+  s.add_development_dependency 'rake', '~> 10.3', '>= 10.3.2'
+  s.add_development_dependency 'coveralls'
 
 
 end

data/spec/cassettes/gzipped_on.yml

@@ -0,0 +1,137 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://www.whatsmyip.org/http-compression-test/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 19 May 2014 22:57:51 GMT
+      Server:
+      - Apache/2.2.24 (Unix) DAV/2 mod_fastcgi/2.4.2 mod_ssl/2.2.24 OpenSSL/0.9.8y
+      Cache-Control:
+      - max-age=1
+      Expires:
+      - Mon, 19 May 2014 22:57:52 GMT
+      X-Powered-By:
+      - Web Server Built Entirely Out of Legos
+      Vary:
+      - Accept-Encoding
+      Content-Encoding:
+      - gzip
+      Content-Length:
+      - '4153'
+      Content-Type:
+      - text/html
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA91ae3PbNrb/W/kUWDYbyRNTsr3pI7GlTuKmTXbz8MTO7e10
+        OhqIhETEJMElQMtK6+++vwOAL1luk/bOnTs3M45I4OC8z8HBAU/+9t3b04uf
+        zp6zxGTp7N69E/87OEkEj2f3BoMTI00qZj8m3OjXm5dnY1Wu2G/sxcXFGTtV
+        WVEKraXK2YXQ5mTigGlZJgxnUcJLLcw0qMwy/CZoJ3KeiWlwKTZrVcY6YJHK
+        jcgBuPooi32WqXjePsVimXIj9sGkKQDaEN1na7FgWpRXotxnvOBRAii1wNqE
+        53FKowZ80e8CA2sZm+QWF7HQUSkLA4QdRnYKKEqWCqPZRlUMNOVywwwUY9nw
+        K5nUbCFkvmoYFTGrNA2QRGzCvDh4qsX0j35ibEkRjdLJJwHMl0Rc5HyRdnGD
+        sTFzEmH8kpUinQYy4ysx12UUsKQUy2kwsSN6omEOMS7y1fYKXhSpCI2qoiSU
+        EenBLhzWC9sZWj3cpteuaEgt+VUNHjCzKYRna3LtCLQo3CRxICNORpiUWj+8
+        ztLA85ZC8hx6AR7yrmnQ98V35+fseyHimgNykicTQjJeE2C2kQUBTtq3Iik6
+        DHjGI60n6/V6nHGZj/FSs23EtZnYd8uONptU6EQIczcKo1SqPwOH878u7Ad+
+        xd1owGBI4P7Q4e6DDmYnEzf/OQg+/LsS5eZPL9eFzHNRfuZ6ghhc8ZLdh8NW
+        KaJnyoLguBldCTOvytSO0iD93R8tqzwiZxjt0fvgV/v//VHwxepjEeyNeRyf
+        plzrUWASqecFPCvYszgHHUhZEGaAL1VU6RGLUsHzuShLVbKd0JQsAK6rRSZN
+        y8N94bjwbPSw73/hoLGsKFUxGsZSI0hFPETuKSvh6QyyzbwURcojcW7gzKPa
+        T8mofT+libAT4CFxNanFa4k7dRK7qYzFe5Becm1Adkt3DduDjvyjGt1ALtmI
+        benriqejPTadkk3Y3sBDNngGJinV2iMK3ij2/t0r9hzpr0QYNogHN/5BpFrc
+        QtFQ1BHPc+S0WpDv1DqvRbklSQfB4P64UNp0ePfW48aAK24XBnv77FdWS/Zk
+        t5w3HZXdj7nhLbUOuUHHfy3UWCNlmVHwW0fkwWAy4dh3jEfUmbhT4NZyO8Tt
+        cTDQa2mihI1qVn4++KUL2oMdRFwLFlgjBU+6EwO44WbO51pVeTwKdB7Pl1ym
+        PTEGuxyNioNRcCKzlU0Kzfaw+uh8YayvVkOGhD0d2vfh7EQXPEfcIVSnQ4sM
+        E8NZ8LCR4PCXhwGSCcBmuzygFHO76Ym5zIvK6C0eF6Xgl+3QbfHbTfgPdRCr
+        XPwlHWyE7mhgUxW75AeQlX+HIz4MqHg4bTj2ajkpZu/zTjFxBnrsXH4UT9iW
+        Htm/np0sytnpH8EetbDn2Kfzle5N/wPTf/8TxvjLKY49ZFDHnOu53agQmp9r
+        7ir/3zR4rjr2zlUhdhk8V79v7zdvL3ba/H/UzGeKalPJU/b/xuColjm4/z+W
+        2Z7m7H1+mUOH7LmtM15wzd5GUVWWjXE/X9M3nZcbTN3rvDXPYxJ1u3Cy/zrb
+        Qm/nfsreCIMT2KXnteazp6+bzsvN7c29GfLVCc4XpbiCs33n7NMUGjfdiouK
+        jiVHXXB3YeWjpK4Pb5U//VqtoXKvQ4v+an2wruT3tSkRBd3a8vc9Z4uxtvy0
+        1nJYO7BjOzDP9Kr2rZriLcv7Rdv2rwW6uSVHr4i7VRw3LJYiU1fi07nslyJd
+        6n3ytxm9g4ffK4yd6WsaPfzbOeV+LtbW/h0y5D102FBLtpZ5rNZjHAKMwtGm
+        qHRi19nidVhjHdZutg3fI8V+RSU4BHs1TWQjdnOHGfp5a4s55rgbeWoLo7gv
+        p+v1KKv7nu+OQJ0Fo11bRu3mpTBVmbPgW8xNKY3S+n4IkFo7p7STie/o4HGh
+        4o09uNGQKO0J7YTXZ37ktlt5MFWr7l73Cq9DJuPpkCawYMJnhCOWVxYZMB/O
+        7mgRYcbSm3hgxxhxcc9hqJNqpNIqy/XQgee8maAzMI2eJEczn8CoJ3JBp25g
+        O7JYuWUvQ6qu5cKzCGW+VCFfqMqEG1VB1tcYZS8xyp7SKPtJVSRNi6IoTYOi
+        UKUJbf0uSqw9wys7d6+6v8qUUbPKlHCxEsgF1lw0L/0FONc2C3bvlsNbGt1i
+        NF+1jEIhxCB++kDrRDVAeJY6jHMdpkpdVgUW/Pji7ctz9oBnxTH77s15f22Z
+        X7ZrxYL6UmFRhiXPLx25H90ge+dGtlSyEi1pWYR4BV3f8hnOXp6xV/6tvyxB
+        xugpxrmLrvXxwr26ReQSF+LasHeCWmnxTqfQSWtRnZBJETxhxqNE5mSicxqz
+        Z9rXbqzPUJSUHYayNKQuJ46bNU+vX7HTZmTLKzLRUrYbQmhkBuNC4TB4foWT
+        o/WsczvJjGIX9fyWVnTSMsF1AnXCCTmSGvGAAfZDPbDFvW5ZoFZNSIUziZCv
+        LGWrvVMqpk/d2JYPrK6b5aVYVSkvQ3Hdc1SL5p1YYdg3TfsoRB5tKwFDCkYM
+        Y2F/WwU8p4lbTlysW5eA98UqCwtkBuol9/Rw5gfv0kUVt45QFRobYIjsS1JA
+        L8tUFoXl5b2bok2bkXq2bBqVfYUaFZIUiAs4c61RWPIUgwgNDLa++tYkotzp
+        pKlcNGjxDDyv5KLk5WY7Jm+pog7N1qHf2Qnmw3PLJ9eiTTxLqTTWF+Gl2MAx
+        0gj2dar8HjPsx+dn7F9is4WgyjtazKXpOfJ7DEByP9BfqE0nDrEDY0+WIBpy
+        KkN5tLFu0Ayzp354yxeu0xqJP30U8lqk283fYXuDwc4I4GnsxDiZYGdxW0xv
+        77HN/KHfzTozPF6oa7v7+K4n15s88k3SCTUieXw0Xim1SgVmYt/UHiOd+1lq
+        pfJYLzYOaLudeiJz7YkFHTDqjTLbOJ4GqKSoXH0i8xQGDhdIo5fH9nLjyddH
+        3xTXx4mQq8Q8eXxQXNt11IsKeRxGqbSXGxEPi2oRHj36+vDx198cPPry4KvH
+        h0ePerA6VYD8+sujg8OvDh89/oaYBGut5HgadRhsq5fu4G+/sZ9/2bN12ehX
+        W5a3ojYlwNb2LnPpFd/TvFYRDrDhojIGmSZRKTJFDfcpgCD9tzBk32P/fYb9
+        jr2Sl4KFIQlEi+FMwXIRlkoZEtWxVovaHqvifab3AbzHULlRA/sDXpcfqCsY
+        j3FWeZ6KDDrWzzYXfPWGZ2Kk934++AWCU2HYBXm2eRmPgAiYXDl3jMrNI4pw
+        7jXCAwLDMciMZYw5Gdtn53VT22I+pmVjeCD10CcT+G4uIoPjYCQWEHScCzMR
+        +fz9+YSnKdzti+vlIkunhw94UbyMp0dfPTo6eAQrPz768uDwEbXmIc+44CVo
+        v0HSGsPsCN9nYolaaeTlJVPejGIVVcQi6manKVTPw5pw+EHr+HKIorXr3h1d
+        OwsFtbdjJIVNAueCvducT+8o2LVa5ESBThl+BOGCmmsaOJLzCMc8UwMnah0S
+        z2CB1OmHl4oiBXkr5jlvPcJ70cVaGrqRu1gLYbwX8c4NlAbTxsHY0LdXb42k
+        fia0o40aLFlb0P85qa1QWKxK+RHM8zSYWfb8dvMptzwT5BUDM2fjLvNILXBb
+        vZ2pvCp+cJH+8LATTF7OVViklaY2m1e1/AjamYhllf0lM9fm+COxKEe1HREK
+        WRuzhaIo8767FWy1IyPmCjUmtIAdbiEe2smtIBxgyEXhsHYBXkjtNwOrSaR+
+        rxEoc9hcgukuOzuTSMMV5RJ2d3QWCvnJxubeqJ9tfbp12bLNvK3bQqsvVGaL
+        XTZhzfCYxncfPAiR3T3v3domixInx3JT52dPSgtriSZpo+KhMnvHIfGogSlm
+        FzitM7pTZVEiokvtrsbdtT8VVloIhuQq7d07hT5VjnSW6zTN7YUNe2lqFFBW
+        9xKdXUl+11cHgOT5xtMLqRRkKSrjinqu9vY/VkI31/+9m3l7JwYgwXgc0yhT
+        S8aZLkQklzJiVBAQ/qVEAEESy9y4brHVGqB4tCmz7Xsxd701DTrk7Iy922aZ
+        MImiBQKgubriqYQCRK1TqjEK6KfBSR0GH0H20X0Y0U65uP3qAJj5dSrylUmm
+        wdGXj5qE5hpM9iVgtqXitt1p4FTA7XGKjjPE/Yx1GPBdIk++fnMc1G8QoMKr
+        W9wI0Uk0bSdr1vVt+Ccpr37r1gqN7M11XNv2aD+Y8Dfe+mrViOrHAtsJCR4k
+        IsVB4RgynZLxgGg8Hvd56JHzLecemwjSOjDuDpRnpVrDB3vBQs8oLxcylWaz
+        FTVv83TDFm6Rdo5ain9X9GlJJy5qt13LNGVwF0TRGAcFBqVAQO1dshNlNvY8
+        Woo3rm3zpRtPLdpxy87tOi3wWGzMzS2djnH/2gXbTwJVyk/E6rOW1XdOfGK3
+        c3dy6lj1vfnabXYa5w9SWY3UEW7w9qzSWA9MWFXWBtD8Clmk+UoJKSdmErCK
+        hpHrY0okMkM+QurHgYhO3gwuEIsridIFBsQi1OxrJiniUPMxXwiCSz1mzyjc
+        jLNpKeKK1qSKx4Sjk09h/ISSkfRpzXAUyRrbAuMZlReUwMjOFUV1odYuA9eW
+        t4lMM9ttF2yZbvbtp1IkG8uqKGHUg/PLFhuW2LuoXjqPlfUmYREn1ERicFyV
+        R8Jmb2IHI+DcsJQIkkqIA4PDr16K0nNg3R1bJNMZal4S66cOEdpOcO6mHXOf
+        0FN6X8krkXsNU4PR4M+yAjU7dlQBgJ5OLwgJtAkSG0KaK2b7B3aPAFPN51pW
+        gTaNQ312M+1uE92dTrAFReiabxoEtMtI3f/ujT4y2/jPyjqbFe0l9S5mRbZx
+        XGlCITLCyCujMm4P1OC5sRveGHWu9tnp+fk++ycKnXN/vIUf/vfrV01Es6cY
+        oAsedv5fPzAXm/tsnUhYlxTOGzmvsxQ2W/bMa5VkPXMpVxX8sOc9nsa+JUqO
+        QwgrrwNKRdSQsrbddwa0CYgSlftEDCa0XqIo9dkvysgVlJcMZbZFalNbTI4P
+        90CIATkdo4nlsxdn1pkSolfQiGW+Ls0gKGZThBYB2i/3CNS2/Fqf79jznNBv
+        LDVrQ0tnBwFC1xCpbzTp3nYrvc2psTScPUjN8bf0naM2yAWjoPvFY4C69dsH
+        K3PsE1oHGTSTcKuQBTRnq0YYYGELRxsDnU8YUT4hxjFPCh06q2GtWKJsoR4C
+        vfSqpIXLL0ivcCVqySMlYWu0pnS5jYNyTiL3kxxSHHKRVj6djHdm3abcd/WQ
+        MvbW4qRK/Td4sweRKjbHRwcHh+ERDrEWDsO9M9nOAwbJjGIKRe6pf7L3AHWT
+        aQcOWVAXzS4+fLSkyoWXK/rIdb5AXYg97FyucvYWZn35nJ0JI7s97Tt4as7q
+        9rhVMxjsuIlpINuNcPT9nruCaYr2uhXWPcQ5tndd7tRQHYwXHiMdIEjJVEd5
+        pTf/TngVS+VquPprClhT0K4yDSjRdLZzDTePhK+u7CWrntCKsVo1H4hafBMa
+        +IR1WfGP/joaaMs+O9a89jm1F7ufxan7BuBPsOoWfg6vjbPbSqPpQ03cnd2J
+        vXGY/QfAuyd8uC0AAA==
+    http_version: 
+  recorded_at: Mon, 19 May 2014 22:57:52 GMT
+recorded_with: VCR 2.5.0

data/spec/http_spec.rb

@@ -6,7 +6,6 @@ require "polipus/page"
 describe Polipus::HTTP do
   
   it 'should download a page' do
-
     VCR.use_cassette('http_test') do
       http = Polipus::HTTP.new
       page = http.fetch_page("http://sfbay.craigslist.org/apa/")
@@ -30,6 +29,7 @@ describe Polipus::HTTP do
   end
 
   describe 'proxy settings' do
+
     it 'should set proxy correctly using a procedure' do
       http = Polipus::HTTP.new({proxy_host: -> con { "127.0.0.0" }, proxy_port: -> con { 8080 }})
       http.proxy_host.should eq "127.0.0.0"
@@ -49,10 +49,25 @@ describe Polipus::HTTP do
       http.proxy_host.should eq "127.0.0.0"
     end
 
+  end
+
+
+  describe 'gzipped content handling' do
+
+    it 'should decode gzip content' do
+      VCR.use_cassette('gzipped_on') do
+        http = Polipus::HTTP.new(gzip_enabled: true, logger: Logger.new(STDOUT))
+        page = http.fetch_page("http://www.whatsmyip.org/http-compression-test/")
+        page.doc.css('.gzip_yes').should_not be_empty
+      end
+    end
+
+
+  end
+
   describe 'staled connections' do
     
     it 'should refresh a staled connection' do
-     
       VCR.use_cassette('http_tconnection_max_hits') do
         http = Polipus::HTTP.new(connection_max_hits: 1, logger: Logger.new(STDOUT))
         http.class.__send__(:attr_reader, :connections)
@@ -65,11 +80,8 @@ describe Polipus::HTTP do
         http.fetch_page("https://www.yahoo.com/tech/expectant-parents-asked-the-internet-to-name-their-83416450388.html")
         http.connections_hits['www.yahoo.com'][443].should be 1
         http.connections['www.yahoo.com'][443].should_not be old_conn
-
       end
     end
   end
 
-  end
-
 end

data/spec/spec_helper.rb

@@ -5,6 +5,10 @@
 #
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 require "digest/md5"
+require "coveralls"
+
+Coveralls.wear!
+
 RSpec.configure do |config|
   config.treat_symbols_as_metadata_keys_with_true_values = true
   config.run_all_when_everything_filtered = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polipus
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Francesco Laurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-13 00:00:00.000000000 Z
+date: 2014-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-bloomfilter
@@ -270,6 +270,40 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.3.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.3'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.3'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! "\n    An easy to use distributed web-crawler framework based on Redis\n
   \ "
 email:
@@ -281,6 +315,8 @@ files:
 - .document
 - .gitignore
 - .rspec
+- .travis.yml
+- AUTHORS
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -328,6 +364,7 @@ files:
 - spec/cassettes/c5ce68499027d490adfbb6e5541881e4.yml
 - spec/cassettes/ce16b11a7df0b70fe90c7f90063fdb8c.yml
 - spec/cassettes/ffe3d588b6df4b9de35e5a7ccaf5a81b.yml
+- spec/cassettes/gzipped_on.yml
 - spec/cassettes/http_tconnection_max_hits.yml
 - spec/cassettes/http_test.yml
 - spec/cassettes/http_test_redirect.yml
@@ -382,6 +419,7 @@ test_files:
 - spec/cassettes/c5ce68499027d490adfbb6e5541881e4.yml
 - spec/cassettes/ce16b11a7df0b70fe90c7f90063fdb8c.yml
 - spec/cassettes/ffe3d588b6df4b9de35e5a7ccaf5a81b.yml
+- spec/cassettes/gzipped_on.yml
 - spec/cassettes/http_tconnection_max_hits.yml
 - spec/cassettes/http_test.yml
 - spec/cassettes/http_test_redirect.yml