jeremyf-anemone 0.1.3

data/lib/anemone.rb

@@ -0,0 +1,2 @@
+require 'rubygems'
+require 'anemone/anemone'

data/lib/anemone/anemone.rb

@@ -0,0 +1,37 @@
+require 'ostruct'
+require 'anemone/core'
+
+module Anemone
+  # Version number
+  VERSION = '0.1.1'
+  
+  # User-Agent string used for HTTP requests
+  USER_AGENT = "Anemone/#{self::VERSION}"
+  
+  #module-wide options
+  def Anemone.options=(options)
+    @options = options
+  end
+  
+  def Anemone.options
+    @options
+  end
+  
+  #
+  # Convenience method to start a crawl using Core
+  #
+  def Anemone.crawl(urls, options = {}, &block)
+    Anemone.options = OpenStruct.new(options)
+	
+    #by default, run 4 Tentacle threads to fetch pages
+    Anemone.options.threads ||= 4
+	
+    #disable verbose output by default
+    Anemone.options.verbose ||= false
+	
+    #by default, don't throw away the page response body after scanning it for links
+    Anemone.options.discard_page_bodies ||= false
+	
+    Core.crawl(urls, &block)
+  end
+end

data/lib/anemone/core.rb

@@ -0,0 +1,211 @@
+require 'net/http'
+require 'thread'
+require 'anemone/tentacle'
+require 'anemone/page_hash'
+
+module Anemone
+  class Core
+    # PageHash storing all Page objects encountered during the crawl
+    attr_reader :pages
+    
+    #
+    # Initialize the crawl with starting *urls* (single URL or Array of URLs)
+    # and optional *block*
+    #
+    def initialize(urls, &block)
+      @urls = [urls].flatten.map{ |url| URI(url) if url.is_a?(String) }
+      @urls.each{ |url| url.path = '/' if url.path.empty? }
+      
+      @tentacles = []
+      @pages = PageHash.new
+      @on_every_page_blocks = []
+      @on_pages_like_blocks = Hash.new { |hash,key| hash[key] = [] }
+      @skip_link_patterns = []
+      @after_crawl_blocks = []
+      
+      block.call(self) if block
+    end
+    
+    #
+    # Convenience method to start a new crawl
+    #
+    def self.crawl(root, &block)
+      self.new(root) do |core|
+        block.call(core) if block
+        core.run
+        return core
+      end
+    end
+    
+    #
+    # Add a block to be executed on the PageHash after the crawl
+    # is finished
+    #
+    def after_crawl(&block)
+      @after_crawl_blocks << block
+      self
+    end
+    
+    #
+    # Add one ore more Regex patterns for URLs which should not be
+    # followed
+    #
+    def skip_links_like(*patterns)
+      if patterns
+        patterns.each do |pattern|
+          @skip_link_patterns << pattern
+        end
+      end
+      self
+    end
+    
+    #
+    # Add a block to be executed on every Page as they are encountered
+    # during the crawl
+    #
+    def on_every_page(&block)
+      @on_every_page_blocks << block
+      self
+    end
+    
+    #
+    # Add a block to be executed on Page objects with a URL matching
+    # one or more patterns
+    #
+    def on_pages_like(*patterns, &block)
+      if patterns
+        patterns.each do |pattern|
+          @on_pages_like_blocks[pattern] << block
+        end
+      end
+      self
+    end
+    
+    #
+    # Specify a block which will select which links to follow on each page.
+    # The block should return an Array of URI objects.
+    #
+    def focus_crawl(&block)
+      @focus_crawl_block = block
+      self
+    end
+    
+    #
+    # Perform the crawl
+    #
+    def run
+      @urls.delete_if { |url| !visit_link?(url) }
+      return if @urls.empty?
+      
+      link_queue = Queue.new
+      page_queue = Queue.new
+
+      Anemone.options.threads.times do |id|
+        @tentacles << Thread.new { Tentacle.new(link_queue, page_queue).run }
+      end
+      
+      @urls.each{ |url| link_queue.enq(url) }
+
+      loop do
+        page = page_queue.deq
+        
+        @pages[page.url] = page
+        
+        puts "#{page.url} Queue: #{link_queue.size}" if Anemone.options.verbose
+        
+        #perform the on_every_page blocks for this page
+        do_page_blocks(page)
+
+        page.doc = nil if Anemone.options.discard_page_bodies
+        
+        links_to_follow(page).each do |link|
+
+          eval(%(def link.from_url; '#{page.url}'; end))
+          link_queue.enq(link)
+          @pages[link] = nil
+        end
+        
+        #create an entry in the page hash for each alias of this page,
+        #i.e. all the pages that redirected to this page
+        page.aliases.each do |aka|
+          if !@pages.has_key?(aka) or @pages[aka].nil?
+            @pages[aka] = page.alias_clone(aka)
+          end
+          @pages[aka].add_alias!(page.url)
+        end
+        
+        # if we are done with the crawl, tell the threads to end
+        if link_queue.empty? and page_queue.empty?
+          until link_queue.num_waiting == @tentacles.size
+            Thread.pass
+          end
+          
+          if page_queue.empty?
+            @tentacles.size.times { |i| link_queue.enq(:END)}
+            break
+          end
+        end
+        
+      end
+
+      @tentacles.each { |t| t.join }
+
+      do_after_crawl_blocks()
+      
+      self
+    end
+    
+    private    
+    
+    #
+    # Execute the after_crawl blocks
+    #
+    def do_after_crawl_blocks
+      @after_crawl_blocks.each {|b| b.call(@pages)}
+    end
+    
+    #
+    # Execute the on_every_page blocks for *page*
+    #
+    def do_page_blocks(page)
+      @on_every_page_blocks.each do |blk|
+        blk.call(page)
+      end
+      
+      @on_pages_like_blocks.each do |pattern, blks|
+        if page.url.to_s =~ pattern
+          blks.each { |blk| blk.call(page) }
+        end
+      end
+    end      
+    
+    #
+    # Return an Array of links to follow from the given page.
+    # Based on whether or not the link has already been crawled,
+    # and the block given to focus_crawl()
+    #
+    def links_to_follow(page)
+      links = @focus_crawl_block ? @focus_crawl_block.call(page) : page.links
+      links.find_all { |link| visit_link?(link) }
+    end
+    
+    #
+    # Returns +true+ if *link* has not been visited already,
+    # and is not excluded by a skip_link pattern. Returns
+    # +false+ otherwise.
+    #
+    def visit_link?(link)
+      !@pages.has_key?(link) and !skip_link?(link)
+    end
+    
+    #
+    # Returns +true+ if *link* should not be visited because
+    # its URL matches a skip_link pattern.
+    #
+    def skip_link?(link)
+      @skip_link_patterns.each { |p| return true if link.path =~ p}
+      return false
+    end
+    
+  end
+end

data/lib/anemone/http.rb

@@ -0,0 +1,38 @@
+require 'net/http'
+
+module Anemone
+  class HTTP < Net::HTTP
+    # Maximum number of redirects to follow on each get_response
+    REDIRECTION_LIMIT = 5
+    
+    #
+    # Retrieve an HTTP response for *url*, following redirects.
+    # Returns the response object, response code, and final URI location.
+    # 
+    def self.get(url)      
+      response = get_response(url)
+      code = Integer(response.code)
+      loc = url
+      
+      limit = REDIRECTION_LIMIT
+      while response.is_a?(Net::HTTPRedirection) and limit > 0
+          loc = URI(response['location'])
+          loc = url.merge(loc) if loc.relative?
+          response = get_response(loc)
+          limit -= 1
+      end
+
+      return response, code, loc
+    end
+    
+    #
+    # Get an HTTPResponse for *url*, sending the appropriate User-Agent string
+    #
+    def self.get_response(url)
+      full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"
+      Net::HTTP.start(url.host, url.port) do |http|
+        return http.get(full_path, {'User-Agent' => Anemone::USER_AGENT })
+      end
+    end
+  end
+end

data/lib/anemone/page.rb

@@ -0,0 +1,180 @@
+require 'anemone/http'
+require 'nokogiri'
+require 'ostruct'
+
+module Anemone
+  class Page
+
+    # The URL of the page
+    attr_reader :url
+    attr_reader :from_url
+
+    # Array of distinct A tag HREFs from the page
+    attr_reader :links
+    # Headers of the HTTP response
+    attr_reader :headers
+    
+    # OpenStruct for user-stored data
+    attr_accessor :data
+    # Nokogiri document for the HTML body
+    attr_accessor :doc
+    # Integer response code of the page
+    attr_accessor :code	
+    # Array of redirect-aliases for the page
+    attr_accessor :aliases
+    # Boolean indicating whether or not this page has been visited in PageHash#shortest_paths!
+    attr_accessor :visited
+    # Used by PageHash#shortest_paths! to store depth of the page
+    attr_accessor :depth
+    
+    #
+    # Create a new Page from the response of an HTTP request to *url*
+    #
+    def self.fetch(url)
+      begin
+        url = URI(url) if url.is_a?(String)
+
+        response, code, location = Anemone::HTTP.get(url)
+
+        aka = nil
+        if !url.eql?(location)
+          aka = location
+        end
+
+        return Page.new(url, response.body, code, response.to_hash, aka, url.respond_to?(:from_url) ? url.from_url : nil)
+      rescue
+        return Page.new(url)
+      end
+    end
+    
+    #
+    # Create a new page
+    #
+    def initialize(url, body = nil, code = nil, headers = nil, aka = nil, from_url = nil)
+      @from_url = from_url
+      @url = url
+      @code = code
+      @headers = headers
+      @links = []
+      @aliases = []
+      @data = OpenStruct.new
+	  
+      @aliases << aka if !aka.nil?
+
+      if body
+        begin
+          @doc = Nokogiri::HTML(body)
+        rescue
+          return
+        end
+
+        return if @doc.nil?
+
+        #get a list of distinct links on the page, in absolute url form
+        @doc.css('a').each do |a| 
+          u = a.attributes['href'].content if a.attributes['href']
+          next if u.nil?
+          
+          begin
+            abs = to_absolute(URI(u))
+          rescue
+            next
+          end
+
+          @links << abs if in_domain?(abs)
+        end
+        
+        @links.uniq!
+      end
+    end
+    
+    
+    #
+    # Return a new page with the same *response* and *url*, but
+    # with a 200 response code
+    #    
+    def alias_clone(url)
+      p = clone
+	  p.add_alias!(@aka) if !@aka.nil?
+	  p.code = 200
+	  p
+    end
+
+    #
+    # Add a redirect-alias String *aka* to the list of the page's aliases
+    #
+    # Returns *self*
+    #
+    def add_alias!(aka)
+      @aliases << aka if !@aliases.include?(aka)
+      self
+    end
+    
+    #
+    # Returns an Array of all links from this page, and all the 
+    # redirect-aliases of those pages, as String objects.
+    #
+    # *page_hash* is a PageHash object with the results of the current crawl.
+    #
+    def links_and_their_aliases(page_hash)
+      @links.inject([]) do |results, link|
+        results.concat([link].concat(page_hash[link].aliases))
+      end
+    end
+    
+    #
+    # The content-type returned by the HTTP request for this page
+    #
+    def content_type
+      @headers['content-type'][0] rescue nil
+    end
+    
+    #
+    # Returns +true+ if the page is a HTML document, returns +false+
+    # otherwise.
+    #
+    def html?
+      (@content_type =~ /text\/html/) == 0
+    end
+    
+    #
+    # Returns +true+ if the page is a HTTP redirect, returns +false+
+    # otherwise.
+    #    
+    def redirect?
+      (300..399).include?(@code)
+    end
+    
+    #
+    # Returns +true+ if the page was not found (returned 404 code),
+    # returns +false+ otherwise.
+    #
+    def not_found?
+      404 == @code
+    end
+    
+    #
+    # Converts relative URL *link* into an absolute URL based on the
+    # location of the page
+    #
+    def to_absolute(link)
+      # remove anchor
+      link = URI.encode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/,''))
+
+      relative = URI(link)
+      absolute = @url.merge(relative)
+
+      absolute.path = '/' if absolute.path.empty?
+
+      return absolute
+    end
+    
+    #
+    # Returns +true+ if *uri* is in the same domain as the page, returns
+    # +false+ otherwise
+    #
+    def in_domain?(uri)
+      uri.host == @url.host
+    end
+  end
+end