anemone 0.0.2 → 0.0.3

data/bin/anemone_url_list.rb

@@ -1,46 +1,54 @@
-#! /usr/bin/env ruby
-# == Synopsis
-#   Crawls a site starting at the given URL, and outputs the URL of each page
-#   in the domain as they are encountered.
-#
-# == Usage
-#   anemone_url_list.rb [options] url
-#
-# == Options
-#   -r, --relative          Output relative URLs (rather than absolute)
-#
-# == Author
-#   Chris Kite
-
-$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
-
-require 'anemone'
-require 'optparse'
-require 'rdoc/usage'
-require 'ostruct'
-
-options = OpenStruct.new
-options.relative = false
-
-# make sure that the last option is a URL we can crawl
-begin
-  URI(ARGV.last)
-rescue
-  RDoc::usage()
-  Process.exit 
-end
-
-# parse command-line options
-opts = OptionParser.new
-opts.on('-r', '--relative') { options.relative = true }
-opts.parse!(ARGV)
-
-Anemone.crawl(ARGV.last) do |anemone|  
-  anemone.on_every_page do |page|
-    if options.relative
-      puts page.url.path
-    else
-      puts page.url
-    end
-  end
-end
+#! /usr/bin/env ruby
+# == Synopsis
+#   Crawls a site starting at the given URL, and outputs the URL of each page
+#   in the domain as they are encountered.
+#
+# == Usage
+#   anemone_url_list.rb [options] url
+#
+# == Options
+#   -r, --relative          Output relative URLs (rather than absolute)
+#
+# == Author
+#   Chris Kite
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'anemone'
+require 'optparse'
+require 'ostruct'
+
+def usage
+  puts <<END
+Usage: anemone_url_list.rb [options] url
+    
+Options:
+  -r, --relative      Output relative URLs (rather than absolute)
+END
+end
+
+options = OpenStruct.new
+options.relative = false
+
+# make sure that the last option is a URL we can crawl
+begin
+  URI(ARGV.last)
+rescue
+  usage
+  Process.exit 
+end
+
+# parse command-line options
+opts = OptionParser.new
+opts.on('-r', '--relative') { options.relative = true }
+opts.parse!(ARGV)
+
+Anemone.crawl(ARGV.last) do |anemone|  
+  anemone.on_every_page do |page|
+    if options.relative
+      puts page.url.path
+    else
+      puts page.url
+    end
+  end
+end

data/bin/anemone_url_list.rb~

@@ -0,0 +1,58 @@
+#! /usr/bin/env ruby
+# == Synopsis
+#   Crawls a site starting at the given URL, and outputs the URL of each page
+#   in the domain as they are encountered.
+#
+# == Usage
+#   anemone_url_list.rb [options] url
+#
+# == Options
+#   -r, --relative          Output relative URLs (rather than absolute)
+#
+# == Author
+#   Chris Kite
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'anemone'
+require 'optparse'
+require 'ostruct'
+
+def usage
+  puts <<END
+Usage: anemone_url_list.rb [options] url
+    
+Options:
+  -r, --relative      Output relative URLs (rather than absolute)
+END
+end
+
+options = OpenStruct.new
+options.relative = false
+
+# make sure that the last option is a URL we can crawl
+begin
+  URI(ARGV.last)
+rescue
+  usage
+  Process.exit 
+end
+
+# parse command-line options
+opts = OptionParser.new
+opts.on('-r', '--relative') { options.relative = true }
+opts.parse!(ARGV)
+
+Anemone.crawl(ARGV.last) do |anemone|  
+  anemone.on_pages_like(/\/about\//, /\/experience\//) do |page|
+    puts "WOOZLE #{page.url}"
+  end
+
+  anemone.on_every_page do |page|
+    if options.relative
+      puts page.url.path
+    else
+      puts page.url
+    end
+  end
+end

data/lib/anemone.rb

@@ -1,2 +1,2 @@
-require 'rubygems'
+require 'rubygems'
 require 'anemone/anemone'

data/lib/anemone/anemone.rb

@@ -1,37 +1,37 @@
-require 'ostruct'
-require 'anemone/core'
-
-module Anemone
-  # Version number
-  VERSION = '0.0.2'
-  
-  # User-Agent string used for HTTP requests
-  USER_AGENT = "Anemone/#{self::VERSION}"
-  
-  #module-wide options
-  def Anemone.options=(options)
-    @options = options
-  end
-  
-  def Anemone.options
-    @options
-  end
-  
-  #
-  # Convenience method to start a crawl using Core
-  #
-  def Anemone.crawl(url, options = {}, &block)
-    Anemone.options = OpenStruct.new(options)
-	
-	#by default, run 4 Tentacle threads to fetch pages
-    Anemone.options.threads ||= 4
-	
-	#disable verbose output by default
-    Anemone.options.verbose ||= false
-	
-	#by default, throw away the page response body after scanning it for links, to save memory
-	Anemone.options.discard_page_bodies ||= true
-	
-    Core.crawl(url, &block)
-  end
+require 'ostruct'
+require 'anemone/core'
+
+module Anemone
+  # Version number
+  VERSION = '0.0.2'
+  
+  # User-Agent string used for HTTP requests
+  USER_AGENT = "Anemone/#{self::VERSION}"
+  
+  #module-wide options
+  def Anemone.options=(options)
+    @options = options
+  end
+  
+  def Anemone.options
+    @options
+  end
+  
+  #
+  # Convenience method to start a crawl using Core
+  #
+  def Anemone.crawl(url, options = {}, &block)
+    Anemone.options = OpenStruct.new(options)
+	
+	#by default, run 4 Tentacle threads to fetch pages
+    Anemone.options.threads ||= 4
+	
+	#disable verbose output by default
+    Anemone.options.verbose ||= false
+	
+	#by default, don't throw away the page response body after scanning it for links
+	Anemone.options.discard_page_bodies ||= false
+	
+    Core.crawl(url, &block)
+  end
 end

data/lib/anemone/core.rb

@@ -1,179 +1,181 @@
-require 'net/http'
-require 'thread'
-require 'anemone/tentacle'
-require 'anemone/page_hash'
-
-module Anemone
-  class Core
-    # PageHash storing all Page objects encountered during the crawl
-    attr_reader :pages
-    
-    #
-    # Initialize the crawl with a starting *url*, *options*, and optional *block*
-    #
-    def initialize(url, &block)
-      url = URI(url) if url.is_a?(String)
-      @url = url
-      @tentacles = []
-      @pages = PageHash.new
-      @on_every_page_blocks = []
-      @on_pages_like_blocks = Hash.new { |hash,key| hash[key] = [] }
-      @skip_link_patterns = []
-      @after_crawl_blocks = []
-      
-      block.call(self) if block
-    end
-    
-    #
-    # Convenience method to start a new crawl
-    #
-    def self.crawl(root, &block)
-      self.new(root) do |core|
-        block.call(core) if block
-        core.run
-        core.do_after_crawl_blocks
-        return core
-      end
-    end
-    
-    #
-    # Add a block to be executed on the PageHash after the crawl
-    # is finished
-    #
-    def after_crawl(&block)
-      @after_crawl_blocks << block
-      self
-    end
-    
-    #
-    # Add one ore more Regex patterns for URLs which should not be
-    # followed
-    #
-    def skip_links_like(*patterns)
-      if patterns
-        patterns.each do |pattern|
-          @skip_link_patterns << pattern
-        end
-      end
-      self
-    end
-    
-    #
-    # Add a block to be executed on every Page as they are encountered
-    # during the crawl
-    #
-    def on_every_page(&block)
-      @on_every_page_blocks << block
-      self
-    end
-    
-    #
-    # Add a block to be executed on Page objects with a URL matching
-    # one or more patterns
-    #
-    def on_pages_like(*patterns, &block)
-      if patterns
-        patterns.each do |pattern|
-          @on_pages_like_blocks[pattern] << block
-        end
-      end
-      self
-    end
-    
-    #
-    # Perform the crawl
-    #
-    def run
-      link_queue = Queue.new
-      page_queue = Queue.new
-
-      Anemone.options.threads.times do |id|
-        @tentacles << Thread.new { Tentacle.new(link_queue, page_queue).run }
-      end
-      
-      return if !visit_link?(@url)
-      
-      link_queue.enq(@url)
-
-      while true do
-        page = page_queue.deq
-        
-        @pages[page.url] = page
-        
-        puts "#{page.url} Queue: #{link_queue.size}" if Anemone.options.verbose
-        
-        do_page_blocks(page)
-        
-        page.links.each do |link| 
-          if visit_link?(link)
-            link_queue.enq(link)
-            @pages[link] = nil
-          end
-        end
-        
-        page.aliases.each do |aka|
-          if !@pages.has_key?(aka) or @pages[aka].nil?
-            @pages[aka] = page.alias_clone(aka)
-          end
-          @pages[aka].add_alias!(page.url)
-        end
-        
-        # if we are done with the crawl, tell the threads to end
-        if link_queue.empty? and page_queue.empty?
-          until link_queue.num_waiting == @tentacles.size
-            Thread.pass
-          end
-          
-          if page_queue.empty?
-            @tentacles.size.times { |i| link_queue.enq(:END)}
-            break
-          end
-        end
-        
-      end
-      
-      @tentacles.each { |t| t.join }
-
-      self
-    end
-    
-    #
-    # Execute the after_crawl blocks
-    #
-    def do_after_crawl_blocks
-      @after_crawl_blocks.each {|b| b.call(@pages)}
-    end
-    
-    #
-    # Execute the on_every_page blocks for *page*
-    #
-    def do_page_blocks(page)
-      @on_every_page_blocks.each do |blk|
-        blk.call(page)
-      end
-      
-      @on_pages_like_blocks.each do |pattern, blk|
-        blk.call(page) if page.url.to_s =~ pattern
-      end
-    end      
-    
-    #
-    # Returns +true+ if *link* has not been visited already,
-    # and is not excluded by a skip_link pattern. Returns
-    # +false+ otherwise.
-    #
-    def visit_link?(link)
-      !@pages.has_key?(link) and !skip_link?(link)
-    end
-    
-    #
-    # Returns +true+ if *link* should not be visited because
-    # its URL matches a skip_link pattern.
-    #
-    def skip_link?(link)
-      @skip_link_patterns.each { |p| return true if link.path =~ p}
-      return false
-    end
-    
-  end
-end
+require 'net/http'
+require 'thread'
+require 'anemone/tentacle'
+require 'anemone/page_hash'
+
+module Anemone
+  class Core
+    # PageHash storing all Page objects encountered during the crawl
+    attr_reader :pages
+    
+    #
+    # Initialize the crawl with a starting *url*, *options*, and optional *block*
+    #
+    def initialize(url, &block)
+      url = URI(url) if url.is_a?(String)
+      @url = url
+      @tentacles = []
+      @pages = PageHash.new
+      @on_every_page_blocks = []
+      @on_pages_like_blocks = Hash.new { |hash,key| hash[key] = [] }
+      @skip_link_patterns = []
+      @after_crawl_blocks = []
+      
+      block.call(self) if block
+    end
+    
+    #
+    # Convenience method to start a new crawl
+    #
+    def self.crawl(root, &block)
+      self.new(root) do |core|
+        block.call(core) if block
+        core.run
+        core.do_after_crawl_blocks
+        return core
+      end
+    end
+    
+    #
+    # Add a block to be executed on the PageHash after the crawl
+    # is finished
+    #
+    def after_crawl(&block)
+      @after_crawl_blocks << block
+      self
+    end
+    
+    #
+    # Add one ore more Regex patterns for URLs which should not be
+    # followed
+    #
+    def skip_links_like(*patterns)
+      if patterns
+        patterns.each do |pattern|
+          @skip_link_patterns << pattern
+        end
+      end
+      self
+    end
+    
+    #
+    # Add a block to be executed on every Page as they are encountered
+    # during the crawl
+    #
+    def on_every_page(&block)
+      @on_every_page_blocks << block
+      self
+    end
+    
+    #
+    # Add a block to be executed on Page objects with a URL matching
+    # one or more patterns
+    #
+    def on_pages_like(*patterns, &block)
+      if patterns
+        patterns.each do |pattern|
+          @on_pages_like_blocks[pattern] << block
+        end
+      end
+      self
+    end
+    
+    #
+    # Perform the crawl
+    #
+    def run
+      link_queue = Queue.new
+      page_queue = Queue.new
+
+      Anemone.options.threads.times do |id|
+        @tentacles << Thread.new { Tentacle.new(link_queue, page_queue).run }
+      end
+      
+      return if !visit_link?(@url)
+      
+      link_queue.enq(@url)
+
+      while true do
+        page = page_queue.deq
+        
+        @pages[page.url] = page
+        
+        puts "#{page.url} Queue: #{link_queue.size}" if Anemone.options.verbose
+        
+        do_page_blocks(page)
+        
+        page.links.each do |link| 
+          if visit_link?(link)
+            link_queue.enq(link)
+            @pages[link] = nil
+          end
+        end
+        
+        page.aliases.each do |aka|
+          if !@pages.has_key?(aka) or @pages[aka].nil?
+            @pages[aka] = page.alias_clone(aka)
+          end
+          @pages[aka].add_alias!(page.url)
+        end
+        
+        # if we are done with the crawl, tell the threads to end
+        if link_queue.empty? and page_queue.empty?
+          until link_queue.num_waiting == @tentacles.size
+            Thread.pass
+          end
+          
+          if page_queue.empty?
+            @tentacles.size.times { |i| link_queue.enq(:END)}
+            break
+          end
+        end
+        
+      end
+      
+      @tentacles.each { |t| t.join }
+
+      self
+    end
+    
+    #
+    # Execute the after_crawl blocks
+    #
+    def do_after_crawl_blocks
+      @after_crawl_blocks.each {|b| b.call(@pages)}
+    end
+    
+    #
+    # Execute the on_every_page blocks for *page*
+    #
+    def do_page_blocks(page)
+      @on_every_page_blocks.each do |blk|
+        blk.call(page)
+      end
+      
+      @on_pages_like_blocks.each do |pattern, blks|
+        if page.url.to_s =~ pattern
+          blks.each { |blk| blk.call(page) }
+        end
+      end
+    end      
+    
+    #
+    # Returns +true+ if *link* has not been visited already,
+    # and is not excluded by a skip_link pattern. Returns
+    # +false+ otherwise.
+    #
+    def visit_link?(link)
+      !@pages.has_key?(link) and !skip_link?(link)
+    end
+    
+    #
+    # Returns +true+ if *link* should not be visited because
+    # its URL matches a skip_link pattern.
+    #
+    def skip_link?(link)
+      @skip_link_patterns.each { |p| return true if link.path =~ p}
+      return false
+    end
+    
+  end
+end