anemone 0.2.3 → 0.3.0

data/lib/anemone/page.rb

@@ -8,39 +8,44 @@ module Anemone
     attr_reader :url
     # Headers of the HTTP response
     attr_reader :headers
-    
+    # URL of the page this one redirected to, if any
+    attr_reader :redirect_to
+    # Exception object, if one was raised during HTTP#fetch_page
+    attr_reader :error
+
     # OpenStruct for user-stored data
     attr_accessor :data
-    # Nokogiri document for the HTML body
-    attr_accessor :doc
     # Integer response code of the page
-    attr_accessor :code	
-    # Array of redirect-aliases for the page
-    attr_accessor :aliases
-    # Boolean indicating whether or not this page has been visited in PageHash#shortest_paths!
+    attr_accessor :code
+    # Boolean indicating whether or not this page has been visited in PageStore#shortest_paths!
     attr_accessor :visited
     # Depth of this page from the root of the crawl. This is not necessarily the
-    # shortest path; use PageHash#shortest_paths! to find that value.
+    # shortest path; use PageStore#shortest_paths! to find that value.
     attr_accessor :depth
     # URL of the page that brought us to this page
     attr_accessor :referer
     # Response time of the request for this page in milliseconds
     attr_accessor :response_time
-    
+
     #
     # Create a new page
     #
-    def initialize(url, body = nil, code = nil, headers = nil, aka = nil, referer = nil, depth = 0, response_time = nil)
+    def initialize(url, params = {})
       @url = url
-      @code = code
-      @headers = headers || {}
-      @headers['content-type'] ||= ['']
-      @aliases = Array(aka)
       @data = OpenStruct.new
-      @referer = referer
-      @depth = depth || 0
-      @response_time = response_time
-      @doc = Nokogiri::HTML(body) if body && html? rescue nil
+
+      @code = params[:code]
+      @headers = params[:headers] || {}
+      @headers['content-type'] ||= ['']
+      @aliases = Array(params[:aka]).compact
+      @referer = params[:referer]
+      @depth = params[:depth] || 0
+      @redirect_to = to_absolute(params[:redirect_to])
+      @response_time = params[:response_time]
+      @body = params[:body]
+      @error = params[:error]
+
+      @fetched = !params[:code].nil?
     end
 
     # Array of distinct A tag HREFs from the page
@@ -48,7 +53,7 @@ module Anemone
       return @links unless @links.nil?
       @links = []
       return @links if !doc
-      
+
       doc.css('a').each do |a|
         u = a.attributes['href'].content rescue nil
         next if u.nil? or u.empty?
@@ -58,52 +63,30 @@ module Anemone
       @links.uniq!
       @links
     end
-    
+
+    # Nokogiri document for the HTML body
+    def doc
+      return @doc if @doc
+      @doc = Nokogiri::HTML(@body) if @body && html? rescue nil
+    end
+
+    # Delete the Nokogiri document and response body to conserve memory
     def discard_doc!
       links # force parsing of page links before we trash the document
-      @doc = nil
-    end
-    
-    #
-    # Return a new page with the same *response* and *url*, but
-    # with a 200 response code
-    #    
-    def alias_clone(url)
-      p = clone
-	  p.add_alias!(@aka) if !@aka.nil?
-	  p.code = 200
-	  p
+      @doc = @body = nil
     end
 
-    #
-    # Add a redirect-alias String *aka* to the list of the page's aliases
-    #
-    # Returns *self*
-    #
-    def add_alias!(aka)
-      @aliases << aka if !@aliases.include?(aka)
-      self
+    def fetched?
+      @fetched
     end
-    
-    #
-    # Returns an Array of all links from this page, and all the 
-    # redirect-aliases of those pages, as String objects.
-    #
-    # *page_hash* is a PageHash object with the results of the current crawl.
-    #
-    def links_and_their_aliases(page_hash)
-      links.inject([]) do |results, link|
-        results.concat([link].concat(page_hash[link].aliases))
-      end
-    end
-    
+
     #
     # The content-type returned by the HTTP request for this page
     #
     def content_type
       headers['content-type'].first
     end
-    
+
     #
     # Returns +true+ if the page is a HTML document, returns +false+
     # otherwise.
@@ -111,15 +94,15 @@ module Anemone
     def html?
       !!(content_type =~ %r{^(text/html|application/xhtml+xml)\b})
     end
-    
+
     #
     # Returns +true+ if the page is a HTTP redirect, returns +false+
     # otherwise.
-    #    
+    #
     def redirect?
       (300..399).include?(@code)
     end
-    
+
     #
     # Returns +true+ if the page was not found (returned 404 code),
     # returns +false+ otherwise.
@@ -127,12 +110,14 @@ module Anemone
     def not_found?
       404 == @code
     end
-    
+
     #
     # Converts relative URL *link* into an absolute URL based on the
     # location of the page
     #
     def to_absolute(link)
+      return nil if link.nil?
+
       # remove anchor
       link = URI.encode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/,''))
 
@@ -143,7 +128,7 @@ module Anemone
 
       return absolute
     end
-    
+
     #
     # Returns +true+ if *uri* is in the same domain as the page, returns
     # +false+ otherwise
@@ -151,5 +136,14 @@ module Anemone
     def in_domain?(uri)
       uri.host == @url.host
     end
+
+    def marshal_dump
+      [@url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched]
+    end
+
+    def marshal_load(ary)
+      @url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched = ary
+    end
+
   end
 end

data/lib/anemone/{page_hash.rb → page_store.rb}

@@ -1,21 +1,52 @@
+require 'forwardable'
+
 module Anemone
-  class PageHash < Hash
-    
+  class PageStore
+    extend Forwardable
+
+    def_delegators :@storage, :keys, :values, :size, :each
+
+    def initialize(storage = {})
+      @storage = storage
+    end
+
     # We typically index the hash with a URI,
     # but convert it to a String for easier retrieval
     def [](index)
-      super(index.to_s)
+      @storage[index.to_s]
     end
-    
+
     def []=(index, other)
-      super(index.to_s, other)
+      @storage[index.to_s] = other
+    end
+
+    def delete(key)
+      @storage.delete key.to_s
     end
-    
+
     def has_key?(key)
-      super(key.to_s)
+      @storage.has_key? key.to_s
+    end
+
+    def each_value
+      each { |key, value| yield value }
+    end
+
+    def values
+      result = []
+      each { |key, value| result << value }
+      result
+    end
+
+    def touch_key(key)
+      self[key] = Page.new(key)
     end
 
-    # Does this PageHash contain the specified URL?
+    def touch_keys(keys)
+      @storage.merge! keys.inject({}) { |h, k| h[k.to_s] = Page.new(k); h }
+    end
+
+    # Does this PageStore contain the specified URL?
     # HTTP and HTTPS versions of a URL are considered to be the same page.
     def has_page?(url)
       schemes = %w(http https)
@@ -24,80 +55,67 @@ module Anemone
         return schemes.any? { |s| u.scheme = s; has_key?(u) }
       end
 
-      has_key?(url)
+      has_key? url
     end
-    
+
     #
     # Use a breadth-first search to calculate the single-source
-    # shortest paths from *root* to all pages in the PageHash
+    # shortest paths from *root* to all pages in the PageStore
     #
     def shortest_paths!(root)
       root = URI(root) if root.is_a?(String)
       raise "Root node not found" if !has_key?(root)
-      
-      each_value {|p| p.visited = false if p}
-      
+
       q = Queue.new
-      
-      q.enq(root)
-      self[root].depth = 0
-      self[root].visited = true
-      while(!q.empty?)
-        url = q.deq
-        
-        next if !has_key?(url)
-        
-        page = self[url]
-        
+
+      q.enq root
+      root_page = self[root]
+      root_page.depth = 0
+      root_page.visited = true
+      self[root] = root_page
+      while !q.empty?
+        page = self[q.deq]
         page.links.each do |u|
-          next if !has_key?(u) or self[u].nil?
-          link = self[u]
-          aliases = [link].concat(link.aliases.map {|a| self[a] })
-          
-          aliases.each do |node|
-            if node.depth.nil? or page.depth + 1 < node.depth
-              node.depth = page.depth + 1
+          begin
+            link = self[u]
+            next if link.nil? || !link.fetched? || link.visited
+
+            q << u unless link.redirect?
+            link.visited = true
+            link.depth = page.depth + 1
+            self[u] = link
+
+            if link.redirect?
+              u = link.redirect_to
+              redo
             end
           end
-          
-          q.enq(self[u].url) if !self[u].visited
-          self[u].visited = true
         end
       end
-      
+
       self
     end
-    
+
     #
-    # Returns a new PageHash by removing redirect-aliases for each
-    # non-redirect Page
+    # Removes all Pages from storage where redirect? is true
     #
-    def uniq
-      results = PageHash.new
-      each do |url, page|
-        #if none of the aliases of this page have been added, and this isn't a redirect page, add this page
-        page_added = page.aliases.inject(false) { |r, a| r ||= results.has_key? a}
-        if !page.redirect? and !page_added
-          results[url] = page.clone 
-          results[url].aliases = []
-        end
-      end
-      
-      results
+    def uniq!
+      each_value { |page| delete page.url if page.redirect? }
+      self
     end
-    
+
     #
     # If given a single URL (as a String or URI), returns an Array of Pages which link to that URL
     # If given an Array of URLs, returns a Hash (URI => [Page, Page...]) of Pages linking to those URLs
     #
     def pages_linking_to(urls)
       unless urls.is_a?(Array)
-        urls = [urls] unless urls.is_a?(Array)
+        urls = [urls]
         single = true
       end
 
       urls.map! do |url|
-        if url.is_a?(String)
+        unless url.is_a?(URI)
           URI(url) rescue nil
         else
           url
@@ -112,7 +130,7 @@ module Anemone
       end
 
       if single and !links.empty?
-        return links.first
+        return links[urls.first]
       else
         return links
       end
@@ -132,11 +150,11 @@ module Anemone
       links.each { |url, pages| links[url] = pages.map{|p| p.url} }
 
       if single and !links.empty?
-        return links.first
+        return links[urls.first]
       else
         return links
-      end	  
+      end
     end
 
   end
-end
+end

data/lib/anemone/storage.rb

@@ -0,0 +1,19 @@
+module Anemone
+  module Storage
+
+    def self.Hash(*args)
+      Hash.new(*args)
+    end
+
+    def self.PStore(*args)
+      require 'anemone/storage/pstore'
+      self::PStore.new(*args)
+    end
+
+    def self.TokyoCabinet(file)
+      require 'anemone/storage/tokyo_cabinet'
+      self::TokyoCabinet.new(file)
+    end
+
+  end
+end

data/lib/anemone/storage/pstore.rb

@@ -0,0 +1,48 @@
+require 'pstore'
+require 'forwardable'
+
+module Anemone
+  module Storage
+    class PStore
+      extend Forwardable
+
+      def_delegators :@keys, :has_key?, :keys, :size
+
+      def initialize(file)
+        File.delete(file) if File.exists?(file)
+        @store = ::PStore.new(file)
+        @keys = {}
+      end
+
+      def [](key)
+        @store.transaction { |s| s[key] }
+      end
+
+      def []=(key,value)
+        @keys[key] = nil
+        @store.transaction { |s| s[key] = value }
+      end
+
+      def delete(key)
+        @keys.delete(key)
+        @store.transaction { |s| s.delete key}
+      end
+
+      def each
+        @keys.each_key do |key|
+          value = nil
+          @store.transaction { |s| value = s[key] }
+          yield key, value
+        end
+      end
+
+      def merge!(hash)
+        @store.transaction do |s|
+          hash.each { |key, value| s[key] = value; @keys[key] = nil }
+        end
+        self
+      end
+
+    end
+  end
+end