anemone 0.1.1 → 0.1.2

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2009 Vertive, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/lib/anemone/anemone.rb

@@ -3,11 +3,8 @@ require 'anemone/core'
 
 module Anemone
   # Version number
-  VERSION = '0.1.1'
-  
-  # User-Agent string used for HTTP requests
-  USER_AGENT = "Anemone/#{self::VERSION}"
-  
+  VERSION = '0.1.2'
+
   #module-wide options
   def Anemone.options=(options)
     @options = options
@@ -31,7 +28,18 @@ module Anemone
 	
     #by default, don't throw away the page response body after scanning it for links
     Anemone.options.discard_page_bodies ||= false
-	
+
+    #by default, identify self as Anemone/VERSION
+    Anemone.options.user_agent ||= "Anemone/#{self::VERSION}"   
+
+    #no delay between requests by default
+    Anemone.options.delay ||= 0
+    
+    #use a single thread if a delay was requested
+    if(Anemone.options.delay != 0)
+      Anemone.options.threads = 1
+    end
+    
     Core.crawl(urls, &block)
   end
 end

data/lib/anemone/http.rb

@@ -31,7 +31,7 @@ module Anemone
     def self.get_response(url)
       full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"
       Net::HTTP.start(url.host, url.port) do |http|
-        return http.get(full_path, {'User-Agent' => Anemone::USER_AGENT })
+        return http.get(full_path, {'User-Agent' => Anemone.options.user_agent })
       end
     end
   end

data/lib/anemone/tentacle.rb

@@ -24,6 +24,8 @@ module Anemone
         page = Page.fetch(link)
         
         @page_queue.enq(page)
+
+        sleep Anemone.options.delay
       end
     end
     

data/spec/anemone_spec.rb

@@ -0,0 +1,36 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Anemone do
+
+  it "should have a version" do
+    Anemone.const_defined?('VERSION').should == true
+  end
+
+  it "should have options" do
+    Anemone.should respond_to(:options)
+  end
+  
+  it "should accept options for the crawl" do
+    Anemone.crawl(SPEC_DOMAIN, :verbose => false, 
+                               :threads => 2, 
+                               :discard_page_bodies => true,
+                               :user_agent => 'test')
+    Anemone.options.verbose.should == false
+    Anemone.options.threads.should == 2
+    Anemone.options.discard_page_bodies.should == true
+    Anemone.options.delay.should == 0
+    Anemone.options.user_agent.should == 'test'
+  end
+  
+  it "should use 1 thread if a delay is requested" do
+    Anemone.crawl(SPEC_DOMAIN, :delay => 0.01, :threads => 2)
+    Anemone.options.threads.should == 1
+  end
+  
+  it "should return a Anemone::Core from the crawl, which has a PageHash" do
+    result = Anemone.crawl(SPEC_DOMAIN)
+    result.should be_an_instance_of(Anemone::Core)
+    result.pages.should be_an_instance_of(Anemone::PageHash)
+  end
+  
+end

data/spec/core_spec.rb

@@ -0,0 +1,128 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+module Anemone
+  describe Core do
+    
+    before(:each) do
+      FakeWeb.clean_registry
+    end
+    
+    it "should crawl all the html pages in a domain by following <a> href's" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1', '2'])
+      pages << FakePage.new('1', :links => ['3'])
+      pages << FakePage.new('2')
+      pages << FakePage.new('3')
+      
+      Anemone.crawl(pages[0].url).should have(4).pages
+    end
+    
+    it "should not leave the original domain" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1'], :hrefs => 'http://www.other.com/')
+      pages << FakePage.new('1')
+      
+      core = Anemone.crawl(pages[0].url)
+      
+      core.should have(2).pages
+      core.pages.keys.map{|k| k.to_s}.should_not include('http://www.other.com/')
+    end
+    
+    it "should follow http redirects" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1'])
+      pages << FakePage.new('1', :redirect => '2')
+      pages << FakePage.new('2')
+      
+      Anemone.crawl(pages[0].url).should have(3).pages     
+    end
+    
+    it "should accept multiple starting URLs" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1'])
+      pages << FakePage.new('1')
+      pages << FakePage.new('2', :links => ['3'])
+      pages << FakePage.new('3')
+      
+      Anemone.crawl([pages[0].url, pages[2].url]).should have(4).pages
+    end
+    
+    it "should include the query string when following links" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1?foo=1'])
+      pages << FakePage.new('1?foo=1')
+      pages << FakePage.new('1')
+      
+      core = Anemone.crawl(pages[0].url)
+      
+      core.should have(2).pages
+      core.pages.keys.map{|k| k.to_s}.should_not include(pages[2].url)
+    end
+    
+    it "should be able to skip links based on a RegEx" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1', '2'])
+      pages << FakePage.new('1')
+      pages << FakePage.new('2')
+      
+      core = Anemone.crawl(pages[0].url) do |a|
+        a.skip_links_like /1/
+      end
+      
+      core.should have(2).pages
+      core.pages.keys.map{|k| k.to_s}.should_not include(pages[1].url)
+    end
+    
+    it "should be able to call a block on every page" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1', '2'])
+      pages << FakePage.new('1')
+      pages << FakePage.new('2')
+      
+      count = 0
+      Anemone.crawl(pages[0].url) do |a|
+        a.on_every_page { count += 1 }
+      end     
+      
+      count.should == 3
+    end
+    
+    it "should not discard page bodies by default" do
+      Anemone.crawl(FakePage.new('0').url).pages.values.first.doc.should_not be_nil
+    end
+    
+    it "should optionally discard page bodies to conserve memory" do
+      core = Anemone.crawl(FakePage.new('0').url, :discard_page_bodies => true)
+      core.pages.values.first.doc.should be_nil
+    end
+    
+    it "should provide a focus_crawl method to select the links on each page to follow" do
+      pages = []
+      pages << FakePage.new('0', :links => ['1', '2'])
+      pages << FakePage.new('1')
+      pages << FakePage.new('2')
+
+      core = Anemone.crawl(pages[0].url) do |a|
+        a.focus_crawl {|p| p.links.reject{|l| l.to_s =~ /1/}}
+      end     
+      
+      core.should have(2).pages
+      core.pages.keys.map{|k| k.to_s}.should_not include(pages[1].url)
+    end
+    
+    it "should optionally delay between page requests" do
+      delay = 0.25
+      
+      pages = []
+      pages << FakePage.new('0', :links => '1')
+      pages << FakePage.new('1')
+      
+      start = Time.now
+      Anemone.crawl(pages[0].url, :delay => delay)
+      finish = Time.now
+      
+      (finish - start).should satisfy {|t| t > delay * 2}
+    end
+    
+  end
+end

data/spec/fakeweb_helper.rb

@@ -0,0 +1,55 @@
+begin
+  require 'fakeweb'
+rescue LoadError
+  warn "You need the 'fakeweb' gem installed to test Anemone"
+  exit
+end
+
+FakeWeb.allow_net_connect = false
+
+module Anemone
+  SPEC_DOMAIN = "http://www.example.com/"
+  
+  class FakePage
+    attr_accessor :links
+    attr_accessor :hrefs
+    
+    def initialize(name = '', options = {})
+      @name = name
+      @links = [options[:links]].flatten if options.has_key?(:links)
+      @hrefs = [options[:hrefs]].flatten if options.has_key?(:hrefs)
+      @redirect = options[:redirect] if options.has_key?(:redirect)
+      
+      create_body
+      add_to_fakeweb
+    end
+    
+    def url
+      SPEC_DOMAIN + @name
+    end
+    
+    private
+    
+    def create_body
+      @body = "<html><body>"
+      @links.each{|l| @body += "<a href=\"#{SPEC_DOMAIN}#{l}\"></a>"} if @links
+      @hrefs.each{|h| @body += "<a href=\"#{h}\"></a>"} if @hrefs
+      @body += "</body></html>"
+    end
+    
+    def add_to_fakeweb
+      options = {:body => @body, :content_type => "text/html", :status => [200, "OK"]}
+      
+      if @redirect
+        options[:status] = [301, "Permanently Moved"] 
+        options[:location] = SPEC_DOMAIN + @redirect
+      end
+
+      FakeWeb.register_uri(:get, SPEC_DOMAIN + @name, options)
+    end
+  end
+end
+
+#default root
+Anemone::FakePage.new
+

data/spec/page_spec.rb

@@ -0,0 +1,49 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+module Anemone
+  describe Page do
+    
+    before(:each) do
+      @page = Page.fetch(FakePage.new('home').url)
+    end
+    
+    it "should be able to fetch a page" do
+      @page.should_not be_nil
+      @page.url.to_s.should include('home')
+    end
+    
+    it "should store the response headers when fetching a page" do
+      @page.headers.should_not be_nil
+      @page.headers.should have_key('content-type')
+    end
+    
+    it "should have an OpenStruct attribute for the developer to store data in" do
+      @page.data.should_not be_nil
+      @page.data.should be_an_instance_of(OpenStruct)
+      
+      @page.data.test = 'test'
+      @page.data.test.should == 'test'
+    end
+    
+    it "should have a Nokogori::HTML::Document attribute for the page body" do
+      @page.doc.should_not be_nil
+      @page.doc.should be_an_instance_of(Nokogiri::HTML::Document)
+    end
+    
+    it "should indicate whether it was fetched after an HTTP redirect" do
+      @page.should respond_to(:redirect?)
+      
+      @page.redirect?.should == false
+      
+      Page.fetch(FakePage.new('redir', :redirect => 'home').url).redirect?.should == true
+    end
+    
+    it "should have a method to tell if a URI is in the same domain as the page" do
+      @page.should respond_to(:in_domain?)
+      
+      @page.in_domain?(URI(FakePage.new('test').url)).should == true
+      @page.in_domain?(URI('http://www.other.com/')).should == false
+    end
+    
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require File.dirname(__FILE__) + '/fakeweb_helper'
+require 'rubygems'
+
+$:.unshift(File.dirname(__FILE__) + '/../lib/')
+require 'anemone'
+
+SPEC_DOMAIN = 'http://www.example.com/'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: anemone
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Chris Kite
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-22 00:00:00 -05:00
+date: 2009-08-10 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -35,23 +35,22 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
-- bin/anemone_pagedepth.rb
-- bin/anemone_url_list.rb
-- bin/anemone_cron.rb
+- LICENSE.txt
+- README.rdoc
 - bin/anemone_count.rb
+- bin/anemone_cron.rb
+- bin/anemone_pagedepth.rb
 - bin/anemone_serialize.rb
-- lib/anemone/tentacle.rb
-- lib/anemone/page.rb
-- lib/anemone/page_hash.rb
+- bin/anemone_url_list.rb
+- lib/anemone.rb
+- lib/anemone/anemone.rb
 - lib/anemone/core.rb
 - lib/anemone/http.rb
-- lib/anemone/anemone.rb
-- lib/anemone.rb
-- README.rdoc
+- lib/anemone/page.rb
+- lib/anemone/page_hash.rb
+- lib/anemone/tentacle.rb
 has_rdoc: true
 homepage: http://anemone.rubyforge.org
-licenses: []
-
 post_install_message: 
 rdoc_options: 
 - -m
@@ -75,9 +74,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: anemone
-rubygems_version: 1.3.4
+rubygems_version: 1.3.1
 signing_key: 
-specification_version: 3
+specification_version: 2
 summary: Anemone web-spider framework
-test_files: []
-
+test_files: 
+- spec/anemone_spec.rb
+- spec/core_spec.rb
+- spec/page_spec.rb
+- spec/fakeweb_helper.rb
+- spec/spec_helper.rb