parolkar-anemone 0.1.2

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2009 Vertive, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,19 @@
+= Anemone
+
+== DESCRIPTION
+Anemone is a web spider framework that can spider a domain and collect useful
+information about the pages it visits. It is versatile, allowing you to
+write your own specialized spider tasks quickly and easily.
+
+== FEATURES
+* Multi-threaded design for high performance
+* Tracks 301 HTTP redirects to understand a page's aliases
+* Built-in BFS algorithm for determining page depth
+* Allows exclusion of URLs based on regular expressions 
+* Can crawl obeying robots.txt
+
+== REQUIREMENTS
+* nokogiri
+
+== EXAMPLES
+See the +bin+ directory for several examples of useful Anemone tasks.

data/bin/anemone_count.rb

@@ -0,0 +1,36 @@
+#! /usr/bin/env ruby
+# == Synopsis
+#   Crawls a site starting at the given URL, and outputs the total number
+#   of unique pages on the site.
+#
+# == Usage
+#   anemone_count.rb url
+#
+# == Author
+#   Chris Kite
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'anemone'
+
+def usage
+  puts <<END
+Usage: anemone_count.rb url
+END
+end
+
+# make sure that the first option is a URL we can crawl
+begin
+  URI(ARGV[0])
+rescue
+  usage
+  Process.exit 
+end
+
+Anemone.crawl(ARGV[0]) do |anemone|
+  anemone.after_crawl do |pages|
+    puts pages.uniq.size
+  end
+end
+
+

data/bin/anemone_cron.rb

@@ -0,0 +1,106 @@
+#! /usr/bin/env ruby
+# == Synopsis
+#   Performs pagedepth, url list, and count functionality
+#   Meant to be run daily as a cron job
+#
+# == Usage
+#   anemone_url_list.rb [options] url
+#
+# == Options
+#   -r, --relative                  Output relative URLs (rather than absolute)
+#   -o, --output filename           Filename to save URL list to. Defaults to urls.txt.
+#
+# == Author
+#   Chris Kite
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'anemone'
+require 'optparse'
+require 'ostruct'
+
+def usage
+  puts <<END
+Usage: anemone_url_list.rb [options] url
+
+Options:
+  -r, --relative           Output relative URLs (rather than absolute)
+  -o, --output filename    Filename to save URL list to. Defautls to urls.txt.
+END
+end
+
+options = OpenStruct.new
+options.relative = false
+options.output_file = 'urls.txt'
+
+# make sure that the last option is a URL we can crawl
+begin
+  URI(ARGV.last)
+rescue
+  usage
+  Process.exit 
+end
+
+# parse command-line options
+opts = OptionParser.new
+opts.on('-r', '--relative')        { options.relative = true }
+opts.on('-o', '--output filename') {|o| options.output_file = o }
+opts.parse!(ARGV)
+
+root = ARGV.last
+
+Anemone.crawl(root, {:discard_page_bodies => true}) do |anemone|  
+  
+  anemone.after_crawl do |pages|
+    puts "Crawl results for #{root}\n"
+    
+    # print a list of 404's
+    not_found = []
+    pages.each_value do |page|
+      url = page.url.to_s
+      not_found << url if page.not_found?
+    end
+    unless not_found.empty?
+      puts "\n404's:"
+
+      missing_links = pages.urls_linking_to(not_found)
+      missing_links.each do |url, links|
+        if options.relative
+          puts URI(url).path.to_s
+        else
+          puts url
+        end
+        links.slice(0..10).each do |u|
+          u = u.path if options.relative
+          puts "  linked from #{u}"
+        end
+        
+        puts " ..." if links.size > 10
+      end
+
+      print "\n"
+    end  
+    
+    # remove redirect aliases, and calculate pagedepths
+    pages = pages.shortest_paths!(root).uniq
+    depths = pages.values.inject({}) do |depths, page|
+      depths[page.depth] ||= 0
+      depths[page.depth] += 1
+      depths
+    end
+    
+    # print the page count
+    puts "Total pages: #{pages.size}\n"
+    
+    # print a list of depths
+    depths.sort.each { |depth, count| puts "Depth: #{depth} Count: #{count}" }
+    
+    # output a list of urls to file
+    file = open(options.output_file, 'w')
+    pages.each_key do |url|
+      url = options.relative ? url.path.to_s : url.to_s
+      file.puts url
+    end
+    
+  end
+end

data/bin/anemone_pagedepth.rb

@@ -0,0 +1,44 @@
+#! /usr/bin/env ruby
+# == Synopsis
+#   Crawls a site starting at the given URL, and outputs a count of
+#   the number of Pages at each depth in the site.
+#
+# == Usage
+#   anemone_pagedepth.rb url
+#
+# == Author
+#   Chris Kite
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'anemone'
+
+def usage
+  puts <<END
+Usage: anemone_pagedepth.rb url
+END
+end
+
+# make sure that the first option is a URL we can crawl
+begin
+  URI(ARGV[0])
+rescue
+  usage
+  Process.exit 
+end
+
+root = ARGV[0]
+Anemone.crawl(root) do |anemone|
+  anemone.skip_links_like %r{^/c/$}, %r{^/stores/$}
+  
+  anemone.after_crawl do |pages|
+    pages = pages.shortest_paths!(root).uniq
+    depths = pages.values.inject({}) do |depths, page|
+      depths[page.depth] ||= 0
+      depths[page.depth] += 1
+      depths
+    end
+    
+    depths.sort.each { |depth, count| puts "Depth: #{depth} Count: #{count}" }
+  end
+end

data/bin/anemone_serialize.rb

@@ -0,0 +1,51 @@
+#! /usr/bin/env ruby
+# == Synopsis
+#   Crawls a site starting at the given URL, and saves the resulting
+#   PageHash object to a file using Marshal serialization.
+#
+# == Usage
+#   anemone_serialize.rb [options] url
+#
+# == Options
+#   -o, --output filename           Filename to save PageHash to. Defaults to crawl.{Time.now}
+#
+# == Author
+#   Chris Kite
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'anemone'
+require 'optparse'
+require 'ostruct'
+
+def usage
+  puts <<END
+Usage: anemone_serialize.rb [options] url
+
+Options:
+  -o, --output filename      Filename to save PageHash to. Defaults to crawl.{Time.now}
+END
+end
+
+# make sure that the first option is a URL we can crawl
+begin
+  URI(ARGV[0])
+rescue
+  usage
+  Process.exit 
+end
+
+options = OpenStruct.new
+options.output_file = "crawl.#{Time.now.to_i}"
+
+# parse command-line options
+opts = OptionParser.new
+opts.on('-o', '--output filename') {|o| options.output_file = o }
+opts.parse!(ARGV)
+
+root = ARGV[0]
+Anemone.crawl(root) do |anemone|
+  anemone.after_crawl do |pages|
+    open(options.output_file, 'w') {|f| Marshal.dump(pages, f)}
+  end
+end

data/bin/anemone_url_list.rb

@@ -0,0 +1,54 @@
+#! /usr/bin/env ruby
+# == Synopsis
+#   Crawls a site starting at the given URL, and outputs the URL of each page
+#   in the domain as they are encountered.
+#
+# == Usage
+#   anemone_url_list.rb [options] url
+#
+# == Options
+#   -r, --relative          Output relative URLs (rather than absolute)
+#
+# == Author
+#   Chris Kite
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'anemone'
+require 'optparse'
+require 'ostruct'
+
+def usage
+  puts <<END
+Usage: anemone_url_list.rb [options] url
+    
+Options:
+  -r, --relative      Output relative URLs (rather than absolute)
+END
+end
+
+options = OpenStruct.new
+options.relative = false
+
+# make sure that the last option is a URL we can crawl
+begin
+  URI(ARGV.last)
+rescue
+  usage
+  Process.exit 
+end
+
+# parse command-line options
+opts = OptionParser.new
+opts.on('-r', '--relative') { options.relative = true }
+opts.parse!(ARGV)
+
+Anemone.crawl(ARGV.last, :discard_page_bodies => true) do |anemone|  
+  anemone.on_every_page do |page|
+    if options.relative
+      puts page.url.path
+    else
+      puts page.url
+    end
+  end
+end

data/lib/anemone.rb

@@ -0,0 +1,2 @@
+require 'rubygems'
+require 'anemone/anemone'

data/lib/anemone/anemone.rb

@@ -0,0 +1,56 @@
+require 'ostruct'
+require 'anemone/core'
+
+module Anemone
+  # Version number
+  VERSION = '0.1.2'
+
+  #module-wide options
+  def Anemone.options=(options)
+    @options = options
+  end
+  
+  def Anemone.options
+    @options
+  end
+  
+  #
+  # Convenience method to start a crawl using Core
+  #
+  def Anemone.crawl(urls, options = {}, &block)
+    Anemone.options = OpenStruct.new(options)
+	
+    #by default, run 4 Tentacle threads to fetch pages
+    Anemone.options.threads ||= 4
+	
+    #disable verbose output by default
+    Anemone.options.verbose ||= false
+	
+    #by default, don't throw away the page response body after scanning it for links
+    Anemone.options.discard_page_bodies ||= false
+
+    #by default, identify self as Anemone/VERSION
+    Anemone.options.user_agent ||= "Anemone/#{self::VERSION}" 
+    
+    #Obey Robots.txt 
+    Anemone.options.obey_robots_dot_txt ||= false 
+    if Anemone.options.obey_robots_dot_txt == true
+      begin      
+        require 'obey_robots_dot_txt'
+      rescue LoadError
+        warn "You need the 'obey_robots_dot_txt' gem installed, (you may run sudo gem install parolkar-obey_robots_dot_txt --source http://gems.github.com )"
+        exit
+      end
+    end  
+    
+    #no delay between requests by default
+    Anemone.options.delay ||= 0
+    
+    #use a single thread if a delay was requested
+    if(Anemone.options.delay != 0)
+      Anemone.options.threads = 1
+    end
+    
+    Core.crawl(urls, &block)
+  end
+end

data/lib/anemone/core.rb

@@ -0,0 +1,209 @@
+require 'net/http'
+require 'thread'
+require 'anemone/tentacle'
+require 'anemone/page_hash'
+
+module Anemone
+  class Core
+    # PageHash storing all Page objects encountered during the crawl
+    attr_reader :pages
+    
+    #
+    # Initialize the crawl with starting *urls* (single URL or Array of URLs)
+    # and optional *block*
+    #
+    def initialize(urls, &block)
+      @urls = [urls].flatten.map{ |url| URI(url) if url.is_a?(String) }
+      @urls.each{ |url| url.path = '/' if url.path.empty? }
+      
+      @tentacles = []
+      @pages = PageHash.new
+      @on_every_page_blocks = []
+      @on_pages_like_blocks = Hash.new { |hash,key| hash[key] = [] }
+      @skip_link_patterns = []
+      @after_crawl_blocks = []
+      
+      block.call(self) if block
+    end
+    
+    #
+    # Convenience method to start a new crawl
+    #
+    def self.crawl(root, &block)
+      self.new(root) do |core|
+        block.call(core) if block
+        core.run
+        return core
+      end
+    end
+    
+    #
+    # Add a block to be executed on the PageHash after the crawl
+    # is finished
+    #
+    def after_crawl(&block)
+      @after_crawl_blocks << block
+      self
+    end
+    
+    #
+    # Add one ore more Regex patterns for URLs which should not be
+    # followed
+    #
+    def skip_links_like(*patterns)
+      if patterns
+        patterns.each do |pattern|
+          @skip_link_patterns << pattern
+        end
+      end
+      self
+    end
+    
+    #
+    # Add a block to be executed on every Page as they are encountered
+    # during the crawl
+    #
+    def on_every_page(&block)
+      @on_every_page_blocks << block
+      self
+    end
+    
+    #
+    # Add a block to be executed on Page objects with a URL matching
+    # one or more patterns
+    #
+    def on_pages_like(*patterns, &block)
+      if patterns
+        patterns.each do |pattern|
+          @on_pages_like_blocks[pattern] << block
+        end
+      end
+      self
+    end
+    
+    #
+    # Specify a block which will select which links to follow on each page.
+    # The block should return an Array of URI objects.
+    #
+    def focus_crawl(&block)
+      @focus_crawl_block = block
+      self
+    end
+    
+    #
+    # Perform the crawl
+    #
+    def run
+      @urls.delete_if { |url| !visit_link?(url) }
+      return if @urls.empty?
+      
+      link_queue = Queue.new
+      page_queue = Queue.new
+
+      Anemone.options.threads.times do |id|
+        @tentacles << Thread.new { Tentacle.new(link_queue, page_queue).run }
+      end
+      
+      @urls.each{ |url| link_queue.enq(url) }
+
+      loop do
+        page = page_queue.deq
+        
+        @pages[page.url] = page
+        
+        puts "#{page.url} Queue: #{link_queue.size}" if Anemone.options.verbose
+        
+        #perform the on_every_page blocks for this page
+        do_page_blocks(page)
+
+        page.doc = nil if Anemone.options.discard_page_bodies
+        
+        links_to_follow(page).each do |link|
+          link_queue.enq(link)
+          @pages[link] = nil
+        end
+        
+        #create an entry in the page hash for each alias of this page,
+        #i.e. all the pages that redirected to this page
+        page.aliases.each do |aka|
+          if !@pages.has_key?(aka) or @pages[aka].nil?
+            @pages[aka] = page.alias_clone(aka)
+          end
+          @pages[aka].add_alias!(page.url)
+        end
+        
+        # if we are done with the crawl, tell the threads to end
+        if link_queue.empty? and page_queue.empty?
+          until link_queue.num_waiting == @tentacles.size
+            Thread.pass
+          end
+          
+          if page_queue.empty?
+            @tentacles.size.times { |i| link_queue.enq(:END)}
+            break
+          end
+        end
+        
+      end
+
+      @tentacles.each { |t| t.join }
+
+      do_after_crawl_blocks()
+      
+      self
+    end
+    
+    private    
+    
+    #
+    # Execute the after_crawl blocks
+    #
+    def do_after_crawl_blocks
+      @after_crawl_blocks.each {|b| b.call(@pages)}
+    end
+    
+    #
+    # Execute the on_every_page blocks for *page*
+    #
+    def do_page_blocks(page)
+      @on_every_page_blocks.each do |blk|
+        blk.call(page)
+      end
+      
+      @on_pages_like_blocks.each do |pattern, blks|
+        if page.url.to_s =~ pattern
+          blks.each { |blk| blk.call(page) }
+        end
+      end
+    end      
+    
+    #
+    # Return an Array of links to follow from the given page.
+    # Based on whether or not the link has already been crawled,
+    # and the block given to focus_crawl()
+    #
+    def links_to_follow(page)
+      links = @focus_crawl_block ? @focus_crawl_block.call(page) : page.links
+      links.find_all { |link| visit_link?(link) }
+    end
+    
+    #
+    # Returns +true+ if *link* has not been visited already,
+    # and is not excluded by a skip_link pattern. Returns
+    # +false+ otherwise.
+    #
+    def visit_link?(link)
+      !@pages.has_key?(link) and !skip_link?(link)
+    end
+    
+    #
+    # Returns +true+ if *link* should not be visited because
+    # its URL matches a skip_link pattern.
+    #
+    def skip_link?(link)
+      @skip_link_patterns.each { |p| return true if link.path =~ p}
+      return false
+    end
+    
+  end
+end

data/lib/anemone/http.rb

@@ -0,0 +1,38 @@
+require 'net/http'
+
+module Anemone
+  class HTTP < Net::HTTP
+    # Maximum number of redirects to follow on each get_response
+    REDIRECTION_LIMIT = 5
+    
+    #
+    # Retrieve an HTTP response for *url*, following redirects.
+    # Returns the response object, response code, and final URI location.
+    # 
+    def self.get(url)      
+      response = get_response(url)
+      code = Integer(response.code)
+      loc = url
+      
+      limit = REDIRECTION_LIMIT
+      while response.is_a?(Net::HTTPRedirection) and limit > 0
+          loc = URI(response['location'])
+          loc = url.merge(loc) if loc.relative?
+          response = (Anemone.options.obey_robots_dot_txt ? (Net::HTTP.get_obeying_robots(loc)) : get_response(loc) )
+          limit -= 1
+      end
+
+      return response, code, loc
+    end
+    
+    #
+    # Get an HTTPResponse for *url*, sending the appropriate User-Agent string
+    #
+    def self.get_response(url)
+      full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"
+      Net::HTTP.start(url.host, url.port) do |http|
+        return http.get(full_path, {'User-Agent' => Anemone.options.user_agent })
+      end
+    end
+  end
+end

data/lib/anemone/page.rb

@@ -0,0 +1,177 @@
+require 'anemone/http'
+require 'nokogiri'
+require 'ostruct'
+
+module Anemone
+  class Page
+
+    # The URL of the page
+    attr_reader :url
+    # Array of distinct A tag HREFs from the page
+    attr_reader :links
+    # Headers of the HTTP response
+    attr_reader :headers
+    
+    # OpenStruct for user-stored data
+    attr_accessor :data
+    # Nokogiri document for the HTML body
+    attr_accessor :doc
+    # Integer response code of the page
+    attr_accessor :code	
+    # Array of redirect-aliases for the page
+    attr_accessor :aliases
+    # Boolean indicating whether or not this page has been visited in PageHash#shortest_paths!
+    attr_accessor :visited
+    # Used by PageHash#shortest_paths! to store depth of the page
+    attr_accessor :depth
+    
+    #
+    # Create a new Page from the response of an HTTP request to *url*
+    #
+    def self.fetch(url)
+      begin
+        url = URI(url) if url.is_a?(String)
+
+        response, code, location = Anemone::HTTP.get(url)
+
+        aka = nil
+        if !url.eql?(location)
+          aka = location
+        end
+
+        return Page.new(url, response.body, code, response.to_hash, aka)
+      rescue
+        return Page.new(url)
+      end
+    end
+    
+    #
+    # Create a new page
+    #
+    def initialize(url, body = nil, code = nil, headers = nil, aka = nil)
+      @url = url
+      @code = code
+      @headers = headers
+      @links = []
+      @aliases = []
+      @data = OpenStruct.new
+	  
+      @aliases << aka if !aka.nil?
+
+      if body
+        begin
+          @doc = Nokogiri::HTML(body)
+        rescue
+          return
+        end
+
+        return if @doc.nil?
+
+        #get a list of distinct links on the page, in absolute url form
+        @doc.css('a').each do |a| 
+          u = a.attributes['href'].content if a.attributes['href']
+          next if u.nil?
+          
+          begin
+            abs = to_absolute(URI(u))
+          rescue
+            next
+          end
+
+          @links << abs if in_domain?(abs)
+        end
+        
+        @links.uniq!
+      end
+    end
+    
+    
+    #
+    # Return a new page with the same *response* and *url*, but
+    # with a 200 response code
+    #    
+    def alias_clone(url)
+      p = clone
+	  p.add_alias!(@aka) if !@aka.nil?
+	  p.code = 200
+	  p
+    end
+
+    #
+    # Add a redirect-alias String *aka* to the list of the page's aliases
+    #
+    # Returns *self*
+    #
+    def add_alias!(aka)
+      @aliases << aka if !@aliases.include?(aka)
+      self
+    end
+    
+    #
+    # Returns an Array of all links from this page, and all the 
+    # redirect-aliases of those pages, as String objects.
+    #
+    # *page_hash* is a PageHash object with the results of the current crawl.
+    #
+    def links_and_their_aliases(page_hash)
+      @links.inject([]) do |results, link|
+        results.concat([link].concat(page_hash[link].aliases))
+      end
+    end
+    
+    #
+    # The content-type returned by the HTTP request for this page
+    #
+    def content_type
+      @headers['content-type'][0] rescue nil
+    end
+    
+    #
+    # Returns +true+ if the page is a HTML document, returns +false+
+    # otherwise.
+    #
+    def html?
+      (@content_type =~ /text\/html/) == 0
+    end
+    
+    #
+    # Returns +true+ if the page is a HTTP redirect, returns +false+
+    # otherwise.
+    #    
+    def redirect?
+      (300..399).include?(@code)
+    end
+    
+    #
+    # Returns +true+ if the page was not found (returned 404 code),
+    # returns +false+ otherwise.
+    #
+    def not_found?
+      404 == @code
+    end
+    
+    #
+    # Converts relative URL *link* into an absolute URL based on the
+    # location of the page
+    #
+    def to_absolute(link)
+      # remove anchor
+      link = URI.encode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/,''))
+
+      relative = URI(link)
+      absolute = @url.merge(relative)
+
+      absolute.path = '/' if absolute.path.empty?
+
+      return absolute
+    end
+    
+    #
+    # Returns +true+ if *uri* is in the same domain as the page, returns
+    # +false+ otherwise
+    #
+    def in_domain?(uri)
+      uri.host == @url.host
+    end
+  end
+end

data/lib/anemone/page_hash.rb

@@ -0,0 +1,116 @@
+module Anemone
+  class PageHash < Hash
+    
+    #
+    # Use a breadth-first search to calculate the single-source
+    # shortest paths from *root* to all pages in the PageHash
+    #
+    def shortest_paths!(root)
+      root = URI(root) if root.is_a?(String)
+      raise "Root node not found" if !has_key?(root)
+      
+      each_value {|p| p.visited = false if p}
+      
+      q = Queue.new
+      
+      q.enq(root)
+      self[root].depth = 0
+      self[root].visited = true
+      while(!q.empty?)
+        url = q.deq
+        
+        next if !has_key?(url)
+        
+        page = self[url]
+        
+        page.links.each do |u|
+          next if !has_key?(u) or self[u].nil?
+          link = self[u]
+          aliases = [link].concat(link.aliases.map {|a| self[a] })
+          
+          aliases.each do |node|
+            if node.depth.nil? or page.depth + 1 < node.depth
+              node.depth = page.depth + 1
+            end
+          end
+          
+          q.enq(self[u].url) if !self[u].visited
+          self[u].visited = true
+        end
+      end
+      
+      self
+    end
+    
+    #
+    # Returns a new PageHash by removing redirect-aliases for each
+    # non-redirect Page
+    #
+    def uniq
+      results = PageHash.new
+      each do |url, page|
+        #if none of the aliases of this page have been added, and this isn't a redirect page, add this page
+        page_added = page.aliases.inject(false) { |r, a| r ||= results.has_key? a}
+        if !page.redirect? and !page_added
+          results[url] = page.clone 
+          results[url].aliases = []
+        end
+      end
+      
+      results
+    end
+    
+    #
+    # If given a single URL (as a String or URI), returns an Array of Pages which link to that URL
+    # If given an Array of URLs, returns a Hash (URI => [Page, Page...]) of Pages linking to those URLs
+    #
+    def pages_linking_to(urls)
+      unless urls.is_a?(Array)
+        urls = [urls] unless urls.is_a?(Array)
+        single = true
+      end
+
+      urls.map! do |url|
+        if url.is_a?(String)
+          URI(url) rescue nil
+        else
+          url
+        end
+      end
+      urls.compact
+
+      links = {}
+      urls.each { |url| links[url] = [] }
+      values.each do |page|
+        urls.each { |url| links[url] << page if page.links.include?(url) }
+      end
+
+      if single and !links.empty?
+        return links.first
+      else
+        return links
+      end
+    end
+
+    #
+    # If given a single URL (as a String or URI), returns an Array of URLs which link to that URL
+    # If given an Array of URLs, returns a Hash (URI => [URI, URI...]) of URLs linking to those URLs
+    #
+    def urls_linking_to(urls)
+      unless urls.is_a?(Array)
+        urls = [urls] unless urls.is_a?(Array)
+        single = true
+      end
+
+      links = pages_linking_to(urls)
+      links.each { |url, pages| links[url] = pages.map{|p| p.url} }
+
+      if single and !links.empty?
+        return links.first
+      else
+        return links
+      end	  
+    end
+
+  end
+end

data/lib/anemone/tentacle.rb

@@ -0,0 +1,33 @@
+require 'anemone/page'
+
+module Anemone
+  class Tentacle
+    
+    #
+    # Create a new Tentacle
+    #
+    def initialize(link_queue, page_queue)
+      @link_queue = link_queue
+      @page_queue = page_queue
+    end
+    
+    #
+    # Gets links from @link_queue, and returns the fetched
+    # Page objects into @page_queue
+    #
+    def run
+      while true do
+        link = @link_queue.deq
+        
+        break if link == :END
+
+        page = Page.fetch(link)
+        
+        @page_queue.enq(page)
+
+        sleep Anemone.options.delay
+      end
+    end
+    
+  end
+end

data/spec/anemone_spec.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Anemone do
+
+  it "should have a version" do
+    Anemone.const_defined?('VERSION').should == true
+  end
+
+  it "should have options" do
+    Anemone.should respond_to(:options)
+  end
+  
+  it "should accept options for the crawl" do
+    Anemone.crawl(SPEC_DOMAIN, :verbose => false, 
+                               :threads => 2, 
+                               :discard_page_bodies => true,
+                               :user_agent => 'test')
+    Anemone.options.verbose.should == false
+    Anemone.options.threads.should == 2
+    Anemone.options.discard_page_bodies.should == true
+    Anemone.options.delay.should == 0
+    Anemone.options.user_agent.should == 'test'
+  end    
+  
+  it "should accept options of obeying Robots.txt for the crawl" do
+    Anemone.crawl(SPEC_DOMAIN, :obey_robots_dot_txt => true)
+    Anemone.options.obey_robots_dot_txt.should == true  
+  end
+  
+  it "should use 1 thread if a delay is requested" do
+    Anemone.crawl(SPEC_DOMAIN, :delay => 0.01, :threads => 2)
+    Anemone.options.threads.should == 1
+  end
+  
+  it "should return a Anemone::Core from the crawl, which has a PageHash" do
+    result = Anemone.crawl(SPEC_DOMAIN)
+    result.should be_an_instance_of(Anemone::Core)
+    result.pages.should be_an_instance_of(Anemone::PageHash)
+  end
+  
+end

data/spec/core_spec.rb

@@ -0,0 +1,128 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+module Anemone
+  describe Core do
+    
+    before(:each) do
+      FakeWeb.clean_registry
+    end
+    
+    it "should crawl all the html pages in a domain by following <a> href's" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1', '2'])
+      pages << FakePage.new('1', :links => ['3'])
+      pages << FakePage.new('2')
+      pages << FakePage.new('3')
+      
+      Anemone.crawl(pages[0].url).should have(4).pages
+    end
+    
+    it "should not leave the original domain" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1'], :hrefs => 'http://www.other.com/')
+      pages << FakePage.new('1')
+      
+      core = Anemone.crawl(pages[0].url)
+      
+      core.should have(2).pages
+      core.pages.keys.map{|k| k.to_s}.should_not include('http://www.other.com/')
+    end
+    
+    it "should follow http redirects" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1'])
+      pages << FakePage.new('1', :redirect => '2')
+      pages << FakePage.new('2')
+      
+      Anemone.crawl(pages[0].url).should have(3).pages     
+    end
+    
+    it "should accept multiple starting URLs" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1'])
+      pages << FakePage.new('1')
+      pages << FakePage.new('2', :links => ['3'])
+      pages << FakePage.new('3')
+      
+      Anemone.crawl([pages[0].url, pages[2].url]).should have(4).pages
+    end
+    
+    it "should include the query string when following links" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1?foo=1'])
+      pages << FakePage.new('1?foo=1')
+      pages << FakePage.new('1')
+      
+      core = Anemone.crawl(pages[0].url)
+      
+      core.should have(2).pages
+      core.pages.keys.map{|k| k.to_s}.should_not include(pages[2].url)
+    end
+    
+    it "should be able to skip links based on a RegEx" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1', '2'])
+      pages << FakePage.new('1')
+      pages << FakePage.new('2')
+      
+      core = Anemone.crawl(pages[0].url) do |a|
+        a.skip_links_like /1/
+      end
+      
+      core.should have(2).pages
+      core.pages.keys.map{|k| k.to_s}.should_not include(pages[1].url)
+    end
+    
+    it "should be able to call a block on every page" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1', '2'])
+      pages << FakePage.new('1')
+      pages << FakePage.new('2')
+      
+      count = 0
+      Anemone.crawl(pages[0].url) do |a|
+        a.on_every_page { count += 1 }
+      end     
+      
+      count.should == 3
+    end
+    
+    it "should not discard page bodies by default" do
+      Anemone.crawl(FakePage.new('0').url).pages.values.first.doc.should_not be_nil
+    end
+    
+    it "should optionally discard page bodies to conserve memory" do
+      core = Anemone.crawl(FakePage.new('0').url, :discard_page_bodies => true)
+      core.pages.values.first.doc.should be_nil
+    end
+    
+    it "should provide a focus_crawl method to select the links on each page to follow" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1', '2'])
+      pages << FakePage.new('1')
+      pages << FakePage.new('2')
+
+      core = Anemone.crawl(pages[0].url) do |a|
+        a.focus_crawl {|p| p.links.reject{|l| l.to_s =~ /1/}}
+      end     
+      
+      core.should have(2).pages
+      core.pages.keys.map{|k| k.to_s}.should_not include(pages[1].url)
+    end
+    
+    it "should optionally delay between page requests" do
+      delay = 0.25
+      
+      pages = []
+      pages << FakePage.new('0', :links => '1')
+      pages << FakePage.new('1')
+      
+      start = Time.now
+      Anemone.crawl(pages[0].url, :delay => delay)
+      finish = Time.now
+      
+      (finish - start).should satisfy {|t| t > delay * 2}
+    end
+    
+  end
+end

data/spec/fakeweb_helper.rb

@@ -0,0 +1,55 @@
+begin
+  require 'fakeweb'
+rescue LoadError
+  warn "You need the 'fakeweb' gem installed to test Anemone"
+  exit
+end
+
+FakeWeb.allow_net_connect = false
+
+module Anemone
+  SPEC_DOMAIN = "http://www.example.com/"
+  
+  class FakePage
+    attr_accessor :links
+    attr_accessor :hrefs
+    
+    def initialize(name = '', options = {})
+      @name = name
+      @links = [options[:links]].flatten if options.has_key?(:links)
+      @hrefs = [options[:hrefs]].flatten if options.has_key?(:hrefs)
+      @redirect = options[:redirect] if options.has_key?(:redirect)
+      
+      create_body
+      add_to_fakeweb
+    end
+    
+    def url
+      SPEC_DOMAIN + @name
+    end
+    
+    private
+    
+    def create_body
+      @body = "<html><body>"
+      @links.each{|l| @body += "<a href=\"#{SPEC_DOMAIN}#{l}\"></a>"} if @links
+      @hrefs.each{|h| @body += "<a href=\"#{h}\"></a>"} if @hrefs
+      @body += "</body></html>"
+    end
+    
+    def add_to_fakeweb
+      options = {:body => @body, :content_type => "text/html", :status => [200, "OK"]}
+      
+      if @redirect
+        options[:status] = [301, "Permanently Moved"] 
+        options[:location] = SPEC_DOMAIN + @redirect
+      end
+
+      FakeWeb.register_uri(:get, SPEC_DOMAIN + @name, options)
+    end
+  end
+end
+
+#default root
+Anemone::FakePage.new
+

data/spec/page_spec.rb

@@ -0,0 +1,49 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+module Anemone
+  describe Page do
+    
+    before(:each) do
+      @page = Page.fetch(FakePage.new('home').url)
+    end
+    
+    it "should be able to fetch a page" do
+      @page.should_not be_nil
+      @page.url.to_s.should include('home')
+    end
+    
+    it "should store the response headers when fetching a page" do
+      @page.headers.should_not be_nil
+      @page.headers.should have_key('content-type')
+    end
+    
+    it "should have an OpenStruct attribute for the developer to store data in" do
+      @page.data.should_not be_nil
+      @page.data.should be_an_instance_of(OpenStruct)
+      
+      @page.data.test = 'test'
+      @page.data.test.should == 'test'
+    end
+    
+    it "should have a Nokogori::HTML::Document attribute for the page body" do
+      @page.doc.should_not be_nil
+      @page.doc.should be_an_instance_of(Nokogiri::HTML::Document)
+    end
+    
+    it "should indicate whether it was fetched after an HTTP redirect" do
+      @page.should respond_to(:redirect?)
+      
+      @page.redirect?.should == false
+      
+      Page.fetch(FakePage.new('redir', :redirect => 'home').url).redirect?.should == true
+    end
+    
+    it "should have a method to tell if a URI is in the same domain as the page" do
+      @page.should respond_to(:in_domain?)
+      
+      @page.in_domain?(URI(FakePage.new('test').url)).should == true
+      @page.in_domain?(URI('http://www.other.com/')).should == false
+    end
+    
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require File.dirname(__FILE__) + '/fakeweb_helper'
+require 'rubygems'
+
+$:.unshift(File.dirname(__FILE__) + '/../lib/')
+require 'anemone'
+
+SPEC_DOMAIN = 'http://www.example.com/'

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification 
+name: parolkar-anemone
+version: !ruby/object:Gem::Version 
+  version: 0.1.2
+platform: ruby
+authors: 
+- Chris Kite
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-05-16 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.3.0
+    version: 
+description: 
+email: 
+executables: 
+- anemone_count.rb
+- anemone_cron.rb
+- anemone_pagedepth.rb
+- anemone_serialize.rb
+- anemone_url_list.rb
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- LICENSE.txt
+- README.rdoc
+- bin/anemone_count.rb
+- bin/anemone_cron.rb
+- bin/anemone_pagedepth.rb
+- bin/anemone_serialize.rb
+- bin/anemone_url_list.rb
+- lib/anemone.rb
+- lib/anemone/anemone.rb
+- lib/anemone/core.rb
+- lib/anemone/http.rb
+- lib/anemone/page.rb
+- lib/anemone/page_hash.rb
+- lib/anemone/tentacle.rb
+has_rdoc: true
+homepage: http://anemone.rubyforge.org
+post_install_message: 
+rdoc_options: 
+- -m
+- README.rdoc
+- -t
+- Anemone
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: anemone
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Anemone web-spider framework
+test_files: 
+- spec/anemone_spec.rb
+- spec/core_spec.rb
+- spec/page_spec.rb
+- spec/fakeweb_helper.rb
+- spec/spec_helper.rb