polipus 0.1.1 → 0.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MTk2M2YyNzJhYjhhY2IxNDUwNThkMjUwZTkzNTgyNjUyMTAxODY0Nw==
+    OGE5NThiNjIyZmFmMDhkMzQ1OGFjMDY3MTgxOThhMDUwMzM4NDE2Zg==
   data.tar.gz: !binary |-
-    MTNkMzZmZDcyODBhNmRjZmMyZjRmOTA3NmM1NGY1OTY3MDhhZGM5ZQ==
+    YjhkYjUzOGI5MjJiYjdjMjQ0ZjdlOTAxOTBlODA0ZjRiYzBhZTI4Ng==
 SHA512:
   metadata.gz: !binary |-
-    MTk3NjZmNmE4MWZmYmI4YzRlMDQxMGI2YTBhN2U2ZjdhMzNjZjQ3ZDk2ODQz
-    OWU0ODQ1ZjMxOGYwZjYyNWE1M2Q2MTE2ZjIxY2E3NjhmZTQwMGNjZTdlZjVm
-    ZTYxYzY4ZjBjNTkyNDY3MDVhODNkMWYwNDE3NjYyZWM0YzhiNjU=
+    NzEzOTk5NDQwMjdmYTZiOTlkNjllYmUxY2M0ZTY1ZWRjOTMwMzEwNjZmZGMy
+    ZjA0NjJiZDgzNzFhYzRkNGQ2ZGY3MmQxMTdlNmJjZDg2MTZiMGE2ZDA0N2Iz
+    NThlMjU2ODllNTIxMjA5OWRhYmY0ZDBlN2I2MjAzNjI3YzVlZDQ=
   data.tar.gz: !binary |-
-    ZTJkMzc2MDI4Yjg5ODMzZmJhMmM5ZmU5YzliMGRlNTliMWI3YWFjNzQ2MWQ1
-    YzBjMzRhZGFlODUyODdjNjExMzFmMWRhZTJiMDhjNjI2OTM0NTQ1NDk4NWE4
-    MzEyN2UxZTIzNjc1OTc2NzBmMTM0MTMwZTgyN2M5NjkxN2IyZDA=
+    ODM0ZDVkNDZlZDQ0NzM2NWEwOGY4MDRkY2IzM2U0MzBiYjQ3YzA0Njk0ZDU5
+    ZTVhYjkwZmQ1ZDRhYjhkMThlOWVkMTNiYzAxNTc1NDUwMDNiMjNmYjE5Nzhm
+    Yzc0MDI5N2ZkMjQxMzEwOTExODUyMmJjNWU1YTdmODAyYmIwNzU=

data/examples/incremental.rb

@@ -0,0 +1,62 @@
+require "polipus"
+require "mongo"
+
+# Define a Mongo connection
+mongo = Mongo::Connection.new(:pool_size => 15, :pool_timeout => 5).db('crawler')
+# Override some default options
+options = {
+  #Redis connection
+  :redis_options => {
+    :host => 'localhost',
+    :db   => 5, 
+    :driver => 'hiredis'
+  },
+  # Page storage: pages is the name of the collection where
+  # pages will be stored
+  :storage    => Polipus::Storage.mongo_store(mongo, 'pages'),
+  # Use your custom user agent
+  :user_agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71",
+  # Use 10 threads
+  :workers    => 20,
+  # Logs goes to the crawler.log file
+  :logger     => Logger.new(STDOUT),
+  # Do not go deeper than 2 levels
+  :depth_limit   => 5,
+
+  # Incremental download:
+  # Set a ttl for each stored page
+  # If a previous stored page is now expired, it will re-downloaded
+  # Mark a page expired after 60s
+  :ttl_page => 60
+}
+
+starting_urls = ["http://rubygems.org/gems"]
+
+# Crawl the entire rubygems's site
+# Polipus.crawler('polipus-rubygems', starting_urls, options)
+
+Polipus.crawler('polipus-rubygems', starting_urls, options) do |crawler|
+  # Ignore urls pointing to a gem file
+  crawler.skip_links_like(/\.gem$/)
+  # Ignore urls pointing to an atom feed
+  crawler.skip_links_like(/\.atom$/)
+  # Ignore urls containing /versions/ path
+  crawler.skip_links_like(/\/versions\//)
+
+  # Adding some metadata to a page
+  # The metadata will be stored on mongo
+  crawler.on_before_save do |page|
+    page.user_data.processed = false
+  end
+
+  # In-place page processing
+  crawler.on_page_downloaded do |page|
+    # A nokogiri object
+    puts "Page title: #{page.doc.css('title').text}" rescue "ERROR"
+  end
+
+  # Do a nifty stuff at the end of the crawling session
+  crawler.on_crawl_end do 
+    # Gong.bang(:loudly)
+  end
+end

data/lib/polipus/http.rb

@@ -36,13 +36,14 @@ module Polipus
             gzip = Zlib::GzipReader.new(StringIO.new(body))    
             body = gzip.read
           end
-          pages << Page.new(location, :body => response.body.dup,
-                                      :code => code,
-                                      :headers => response.to_hash,
-                                      :referer => referer,
-                                      :depth => depth,
-                                      :redirect_to => redirect_to,
-                                      :response_time => response_time)
+          pages << Page.new(location, :body          => response.body.dup,
+                                      :code          => code,
+                                      :headers       => response.to_hash,
+                                      :referer       => referer,
+                                      :depth         => depth,
+                                      :redirect_to   => redirect_to,
+                                      :response_time => response_time,
+                                      :fetched_at    => Time.now.to_i)
         end
 
         return pages

data/lib/polipus/page.rb

@@ -35,6 +35,8 @@ module Polipus
     # Default: true  
     attr_accessor :storable
 
+    attr_accessor :fetched_at
+
     #
     # Create a new page
     #
@@ -54,6 +56,7 @@ module Polipus
       @user_data = OpenStruct.new
       @domain_aliases = params[:domain_aliases] ||= []
       @storable = true
+      @fetched_at = params[:fetched_at]
     end
 
     #
@@ -177,17 +180,19 @@ module Polipus
     end
 
     def to_hash
-      {'url' => @url.to_s,
-       'headers' => Marshal.dump(@headers),
-       'body' => @body,
-       'links' => links.map(&:to_s), 
-       'code' => @code,
-       'depth' => @depth,
-       'referer' => @referer.to_s,
-       'redirect_to' => @redirect_to.to_s,
+      {
+       'url'           => @url.to_s,
+       'headers'       => Marshal.dump(@headers),
+       'body'          => @body,
+       'links'         => links.map(&:to_s), 
+       'code'          => @code,
+       'depth'         => @depth,
+       'referer'       => @referer.to_s,
+       'redirect_to'   => @redirect_to.to_s,
        'response_time' => @response_time,
-       'fetched' => @fetched,
-       'user_data' => @user_data.nil? ? {} : @user_data.marshal_dump
+       'fetched'       => @fetched,
+       'user_data'     => @user_data.nil? ? {} : @user_data.marshal_dump,
+       'fetched_at'    => @fetched_at
      }
     end
 
@@ -198,22 +203,34 @@ module Polipus
       th.to_json
     end
 
+    #
+    # Returns +true+ if page is marked as storeable
+    # +false+ otherwise
+    # Default is +true+
+    #
     def storable?
       @storable
     end
 
+    def expired? ttl
+      return false if fetched_at.nil?
+      (Time.now.to_i - ttl) > fetched_at
+    end
+
     def self.from_hash(hash)
       page = self.new(URI(hash['url']))
-      {'@headers' => hash['headers'] ? Marshal.load(hash['headers']) : {'content-type' => ['']},
-       '@body'    => hash['body'],
-       '@links'   => hash['links'] ? hash['links'].map { |link| URI(link) } : [],
-       '@code'    => hash['code'].to_i,
-       '@depth'   => hash['depth'].to_i,
-       '@referer' => hash['referer'],
-       '@redirect_to' => (!!hash['redirect_to'] && !hash['redirect_to'].empty?) ? URI(hash['redirect_to']) : nil,
-       '@response_time' => hash['response_time'].to_i,
-       '@fetched' => hash['fetched'],
-       '@user_data' => hash['user_data'] ? OpenStruct.new(hash['user_data']) : nil
+      {
+        '@headers'       => hash['headers'] ? Marshal.load(hash['headers']) : {'content-type' => ['']},
+        '@body'          => hash['body'],
+        '@links'         => hash['links'] ? hash['links'].map { |link| URI(link) } : [],
+        '@code'          => hash['code'].to_i,
+        '@depth'         => hash['depth'].to_i,
+        '@referer'       => hash['referer'],
+        '@redirect_to'   => (!!hash['redirect_to'] && !hash['redirect_to'].empty?) ? URI(hash['redirect_to']) : nil,
+        '@response_time' => hash['response_time'].to_i,
+        '@fetched'       => hash['fetched'],
+        '@user_data'     => hash['user_data'] ? OpenStruct.new(hash['user_data']) : nil,
+        '@fetched_at'    => hash['fetched_at']
       }.each do |var, value|
         page.instance_variable_set(var, value)
       end

data/lib/polipus/storage/memory_store.rb

@@ -0,0 +1,56 @@
+require "thread"
+module Polipus
+  module Storage
+    class MemoryStore < Base
+
+      def initialize(options = {})
+        @store = Hash.new
+        @semaphore = Mutex.new
+      end
+
+      def add page
+        @semaphore.synchronize {
+          u = uuid(page)
+          @store[u] = page
+          u
+        }
+      end
+
+      def exists?(page)
+        @semaphore.synchronize {
+          @store.key?(uuid(page))
+        }
+      end
+
+      def get page
+        @semaphore.synchronize {
+          @store[uuid(page)]
+        }
+      end
+
+      def remove page
+        @semaphore.synchronize {
+          @store.delete(uuid(page))
+        }
+      end
+
+      def count
+        @semaphore.synchronize {
+          @store.count
+        }
+      end
+
+      def each
+        @store.each do |k,v|
+          yield k,v
+        end
+      end
+
+      def clear
+        @semaphore.synchronize {
+          @store = Hash.new
+        }
+      end
+    end
+  end
+end

data/lib/polipus/storage/mongo_store.rb

@@ -39,9 +39,7 @@ module Polipus
       def get page
         @semaphore.synchronize {
           data = @mongo[@collection].find({:uuid => uuid(page)}).limit(1).first
-          if data
-            return load_page(data)
-          end
+          return load_page(data) if data
         }
       end
 
@@ -75,7 +73,11 @@ module Polipus
           end
           begin
             hash['body'] = Zlib::Inflate.inflate(hash['body']) if @compress_body && hash['body'] && !hash['body'].empty?
-            return Page.from_hash(hash)
+            page = Page.from_hash(hash)
+            if page.fetched_at.nil?
+              page.fetched_at = hash['_id'].generation_time.to_i
+            end
+            return page
           rescue
           end
           nil

data/lib/polipus/storage.rb

@@ -27,5 +27,10 @@ module Polipus
       require 'polipus/storage/dev_null'
       self::DevNull.new
     end
+
+    def self.memory_store
+      require 'polipus/storage/memory_store'
+      self::MemoryStore.new
+    end
   end
 end

data/lib/polipus/version.rb

@@ -1,4 +1,4 @@
 module Polipus
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
   HOMEPAGE = "https://github.com/taganaka/polipus"
 end

data/lib/polipus.rb

@@ -23,7 +23,7 @@ module Polipus
     OPTS = {
       # run 4 threads
       :workers => 4,
-      # identify self as Anemone/VERSION
+      # identify self as Polipus/VERSION
       :user_agent => "Polipus - #{Polipus::VERSION} - #{Polipus::HOMEPAGE}",
       # by default, don't limit the depth of the crawl
       :depth_limit => false,
@@ -39,12 +39,17 @@ module Polipus
       :read_timeout => 30,
       # HTTP open connection timeout in seconds
       :open_timeout => 10,
+      # Time to wait for new messages on Redis
+      # After this timeout, current crawling session is marked as terminated
+      :queue_timeout => 30,
       # An URL tracker instance. default is Bloomfilter based on redis
       :url_tracker => nil,
       # A Redis options {} that will be passed directly to Redis.new
       :redis_options => {},
       # An instance of logger
       :logger => nil,
+      # A logger level
+      :logger_level => nil,
       # whether the query string should be included in the saved page
       :include_query_string_in_saved_page => true,
       # Max number of items to keep on redis
@@ -62,7 +67,9 @@ module Polipus
       # Eg It can be used to follow links with and without 'www' domain
       :domain_aliases => [],
       # Mark a connection as staled after connection_max_hits request
-      :connection_max_hits => nil
+      :connection_max_hits => nil,
+      # Page TTL: mark a page as expired after ttl_page seconds
+      :ttl_page => nil
     }
 
     attr_reader :storage
@@ -86,6 +93,7 @@ module Polipus
 
       @job_name     = job_name
       @options      = OPTS.merge(options)
+      @options[:queue_timeout] = 1 if @options[:queue_timeout] <= 0
       @logger       = @options[:logger]  ||= Logger.new(nil)
       
       unless @logger.class.to_s == "Log4r::Logger"
@@ -137,8 +145,9 @@ module Polipus
 
       q = queue_factory
       @urls.each do |u|
-        next if url_tracker.visited?(u.to_s)
-        q << Page.new(u.to_s, :referer => '').to_json
+        page = Page.new(u.to_s, :referer => '')
+        page.user_data.p_seeded = true
+        q << page.to_json
       end
 
       return if q.empty?
@@ -149,7 +158,7 @@ module Polipus
           @logger.debug {"Start worker #{worker_number}"}
           http  = @http_pool[worker_number]   ||= HTTP.new(@options)
           queue = @queues_pool[worker_number] ||= queue_factory
-          queue.process(false, @options[:read_timeout]) do |message|
+          queue.process(false, @options[:queue_timeout]) do |message|
 
             next if message.nil?
 
@@ -163,7 +172,7 @@ module Polipus
               next
             end
 
-            if @storage.exists? page
+            if page_exists? page
               @logger.info {"[worker ##{worker_number}] Page [#{page.url.to_s}] already stored."}
               queue.commit
               next
@@ -180,7 +189,7 @@ module Polipus
               @logger.info {"Got redirects! #{rurls}"}
               page = pages.pop
               page.aliases = pages.collect { |e| e.url }
-              if @storage.exists?(page)
+              if page_exists? page
                 @logger.info {"[worker ##{worker_number}] Page [#{page.url.to_s}] already stored."}
                 queue.commit
                 next
@@ -202,7 +211,7 @@ module Polipus
             end
             
             if page
-              @logger.debug {"[worker ##{worker_number}] Fetched page: [#{page.url.to_s}] Referer: [#{page.referer}] Depth: [#{page.depth}] Code: [#{page.code}] Response Time: [#{page.response_time}]"}
+              @logger.debug {"[worker ##{worker_number}] Fetched page: [#{page.url.to_s}] Referrer: [#{page.referer}] Depth: [#{page.depth}] Code: [#{page.code}] Response Time: [#{page.response_time}]"}
               @logger.info  {"[worker ##{worker_number}] Page [#{page.url.to_s}] downloaded"}
             end
             
@@ -264,7 +273,7 @@ module Polipus
       self
     end
 
-    # A block of code will be executed on every page donloaded
+    # A block of code will be executed on every page downloaded
     # before being saved in the registered storage
     def on_before_save(&block)
       @on_before_save << block
@@ -272,7 +281,7 @@ module Polipus
     end
 
     # A block of code will be executed
-    # on every page donloaded. The code is used to extract urls to visit
+    # on every page downloaded. The code is used to extract urls to visit
     # see links_for method
     def focus_crawl(&block)
       @focus_crawl_block = block
@@ -332,6 +341,11 @@ module Polipus
       # URLs enqueue policy
       def should_be_visited?(url, with_tracker = true)
 
+        # return +true+ If an url is part of the initial seeder
+        # no matter what
+
+        return true if @urls.map(&:to_s).include?(url.to_s)
+
         # Check against whitelist pattern matching
         unless @follow_links_like.empty?
           return false unless @follow_links_like.any?{|p| url.path =~ p}  
@@ -342,9 +356,12 @@ module Polipus
           return false if @skip_links_like.any?{|p| url.path =~ p}
         end
 
+        #Page is marked as expired
+        return true if page_expired?(Page.new(url))
+
         # Check against url tracker
         if with_tracker
-          return false if  url_tracker.visited?(@options[:include_query_string_in_saved_page] ? url.to_s : url.to_s.gsub(/\?.*$/,''))
+          return false if url_tracker.visited?(@options[:include_query_string_in_saved_page] ? url.to_s : url.to_s.gsub(/\?.*$/,''))
         end
         true
       end
@@ -356,6 +373,19 @@ module Polipus
         links
       end
 
+      def page_expired? page
+        return false if @options[:ttl_page].nil?
+        stored_page = @storage.get(page)
+        r = stored_page && stored_page.expired?(@options[:ttl_page])
+        @logger.debug {"Page #{page.url.to_s} marked as expired"} if r
+        r
+      end
+
+      def page_exists? page
+        return false if page.user_data && page.user_data.p_seeded
+        @storage.exists?(page) && !page_expired?(page)
+      end
+
       # The url is enqueued for a later visit
       def enqueue url_to_visit, current_page, queue
         page_to_visit = Page.new(url_to_visit.to_s, :referer => current_page.url.to_s, :depth => current_page.depth + 1)