anemone 0.0.6 → 0.1.0

data/lib/anemone/anemone.rb

@@ -3,7 +3,7 @@ require 'anemone/core'
 
 module Anemone
   # Version number
-  VERSION = '0.0.6'
+  VERSION = '0.1.0'
   
   # User-Agent string used for HTTP requests
   USER_AGENT = "Anemone/#{self::VERSION}"
@@ -20,7 +20,7 @@ module Anemone
   #
   # Convenience method to start a crawl using Core
   #
-  def Anemone.crawl(url, options = {}, &block)
+  def Anemone.crawl(urls, options = {}, &block)
     Anemone.options = OpenStruct.new(options)
 	
     #by default, run 4 Tentacle threads to fetch pages
@@ -32,6 +32,6 @@ module Anemone
     #by default, don't throw away the page response body after scanning it for links
     Anemone.options.discard_page_bodies ||= false
 	
-    Core.crawl(url, &block)
+    Core.crawl(urls, &block)
   end
 end

data/lib/anemone/core.rb

@@ -9,12 +9,13 @@ module Anemone
     attr_reader :pages
     
     #
-    # Initialize the crawl with a starting *url*, *options*, and optional *block*
+    # Initialize the crawl with starting *urls* (single URL or Array of URLs)
+    # and optional *block*
     #
-    def initialize(url, &block)
-      url = URI(url) if url.is_a?(String)
-      @url = url
-      @url.path = "/" if @url.path.empty?
+    def initialize(urls, &block)
+      @urls = [urls].flatten.map{ |url| URI(url) if url.is_a?(String) }
+      @urls.each{ |url| url.path = '/' if url.path.empty? }
+      
       @tentacles = []
       @pages = PageHash.new
       @on_every_page_blocks = []
@@ -80,10 +81,22 @@ module Anemone
       self
     end
     
+    #
+    # Specify a block which will select which links to follow on each page.
+    # The block should return an Array of URI objects.
+    #
+    def focus_crawl(&block)
+      @focus_crawl_block = block
+      self
+    end
+    
     #
     # Perform the crawl
     #
     def run
+      @urls.delete_if { |url| !visit_link?(url) }
+      return if @urls.empty?
+      
       link_queue = Queue.new
       page_queue = Queue.new
 
@@ -91,28 +104,27 @@ module Anemone
         @tentacles << Thread.new { Tentacle.new(link_queue, page_queue).run }
       end
       
-      return if !visit_link?(@url)
-      
-      link_queue.enq(@url)
+      @urls.each{ |url| link_queue.enq(url) }
 
-      while true do
+      loop do
         page = page_queue.deq
         
         @pages[page.url] = page
         
         puts "#{page.url} Queue: #{link_queue.size}" if Anemone.options.verbose
         
+        #perform the on_every_page blocks for this page
         do_page_blocks(page)
 
         page.doc = nil if Anemone.options.discard_page_bodies
         
-        page.links.each do |link| 
-          if visit_link?(link)
-            link_queue.enq(link)
-            @pages[link] = nil
-          end
+        links_to_follow(page).each do |link|
+          link_queue.enq(link)
+          @pages[link] = nil
         end
         
+        #create an entry in the page hash for each alias of this page,
+        #i.e. all the pages that redirected to this page
         page.aliases.each do |aka|
           if !@pages.has_key?(aka) or @pages[aka].nil?
             @pages[aka] = page.alias_clone(aka)
@@ -165,6 +177,16 @@ module Anemone
       end
     end      
     
+    #
+    # Return an Array of links to follow from the given page.
+    # Based on whether or not the link has already been crawled,
+    # and the block given to focus_crawl()
+    #
+    def links_to_follow(page)
+      links = @focus_crawl_block ? @focus_crawl_block.call(page) : page.links
+      links.find_all { |link| visit_link?(link) }
+    end
+    
     #
     # Returns +true+ if *link* has not been visited already,
     # and is not excluded by a skip_link pattern. Returns

data/lib/anemone/page.rb

@@ -9,12 +9,12 @@ module Anemone
     attr_reader :url
     # Array of distinct A tag HREFs from the page
     attr_reader :links
-    #Content-type of the  HTTP response
-    attr_reader :content_type
+    # Headers of the HTTP response
+    attr_reader :headers
     
-    #OpenStruct for user-stored data
+    # OpenStruct for user-stored data
     attr_accessor :data
-    #Nokogiri document for the HTML body
+    # Nokogiri document for the HTML body
     attr_accessor :doc
     # Integer response code of the page
     attr_accessor :code	
@@ -39,7 +39,7 @@ module Anemone
           aka = location
         end
 
-        return Page.new(url, response.body, code, response['Content-Type'], aka)
+        return Page.new(url, response.body, code, response.to_hash, aka)
       rescue
         return Page.new(url)
       end
@@ -48,10 +48,10 @@ module Anemone
     #
     # Create a new page
     #
-    def initialize(url, body = nil, code = nil, content_type = nil, aka = nil)
+    def initialize(url, body = nil, code = nil, headers = nil, aka = nil)
       @url = url
       @code = code
-      @content_type = content_type
+      @headers = headers
       @links = []
       @aliases = []
       @data = OpenStruct.new
@@ -119,6 +119,13 @@ module Anemone
       end
     end
     
+    #
+    # The content-type returned by the HTTP request for this page
+    #
+    def content_type
+      @headers['content-type'][0] rescue nil
+    end
+    
     #
     # Returns +true+ if the page is a HTML document, returns +false+
     # otherwise.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: anemone
 version: !ruby/object:Gem::Version 
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors: 
 - Chris Kite
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-04 00:00:00 -05:00
+date: 2009-07-11 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -35,23 +35,22 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
+- bin/anemone_url_list.rb
+- bin/anemone_serialize.rb
+- bin/anemone_pagedepth.rb
 - bin/anemone_count.rb
 - bin/anemone_cron.rb
-- bin/anemone_pagedepth.rb
-- bin/anemone_serialize.rb
-- bin/anemone_url_list.rb
-- lib/anemone/anemone.rb
+- lib/anemone.rb
+- lib/anemone
+- lib/anemone/page.rb
 - lib/anemone/core.rb
+- lib/anemone/anemone.rb
 - lib/anemone/http.rb
-- lib/anemone/page.rb
-- lib/anemone/page_hash.rb
 - lib/anemone/tentacle.rb
-- lib/anemone.rb
+- lib/anemone/page_hash.rb
 - README.rdoc
 has_rdoc: true
 homepage: http://anemone.rubyforge.org
-licenses: []
-
 post_install_message: 
 rdoc_options: 
 - -m
@@ -75,9 +74,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: anemone
-rubygems_version: 1.3.4
+rubygems_version: 1.3.1
 signing_key: 
-specification_version: 3
+specification_version: 2
 summary: Anemone web-spider framework
 test_files: []