spk-anemone 0.3.1 → 0.4.0

data/CHANGELOG.rdoc

@@ -1,3 +1,15 @@
+== 0.4.0 / 2010-04-08
+
+* Major enchancements
+
+  * Cookies can be accepted and sent with each HTTP request.
+
+== 0.3.2 / 2010-02-04
+
+* Bug fixes
+
+  * Fixed issue that allowed following redirects off the original domain
+
 == 0.3.1 / 2010-01-22
 
 * Minor enhancements

data/lib/anemone/cookie_store.rb

@@ -0,0 +1,35 @@
+require 'delegate'
+require 'webrick/cookie'
+
+class WEBrick::Cookie
+  def expired?
+    !!expires && expires < Time.now
+  end
+end
+
+module Anemone
+  class CookieStore < DelegateClass(Hash)
+
+    def initialize(cookies = nil)
+      @cookies = {}
+      cookies.each { |name, value| @cookies[name] = WEBrick::Cookie.new(name, value) } if cookies
+      super(@cookies)
+    end
+
+    def merge!(set_cookie_str)
+      begin
+        cookie_hash = WEBrick::Cookie.parse_set_cookies(set_cookie_str).inject({}) do |hash, cookie|
+          hash[cookie.name] = cookie if !!cookie
+          hash
+        end
+        @cookies.merge! cookie_hash
+      rescue
+      end
+    end
+
+    def to_s
+      @cookies.values.reject { |cookie| cookie.expired? }.map { |cookie| "#{cookie.name}=#{cookie.value}" }.join(';')
+    end
+
+  end
+end

data/lib/anemone/core.rb

@@ -7,7 +7,7 @@ require 'anemone/storage'
 
 module Anemone
 
-  VERSION = '0.3.1';
+  VERSION = '0.4.0';
 
   #
   # Convenience method to start a crawl
@@ -42,14 +42,18 @@ module Anemone
       :redirect_limit => 5,
       # storage engine defaults to Hash in +process_options+ if none specified
       :storage => nil,
+      # Hash of cookie name => value to send with HTTP requests
+      :cookies => nil,
+      # accept cookies from the server and send them back?
+      :accept_cookies => false,
       # Authentication
       :authorization => nil,
     }
 
     # Create setter methods for all options to be called from the crawl block
     DEFAULT_OPTS.keys.each do |key|
-      define_method "#{key}=" do |*args|
-        @opts[key.to_sym] = *args
+      define_method "#{key}=" do |value|
+        @opts[key.to_sym] = value
       end
     end
 
@@ -178,7 +182,7 @@ module Anemone
         end
       end
 
-      @tentacles.each { |t| t.join }
+      @tentacles.each { |thread| thread.join }
       do_after_crawl_blocks
       self
     end
@@ -191,6 +195,18 @@ module Anemone
       @opts[:threads] = 1 if @opts[:delay] > 0
       @pages = PageStore.new(@opts[:storage] || Anemone::Storage.Hash)
       @robots = Robots.new(@opts[:user_agent]) if @opts[:obey_robots_txt]
+
+      freeze_options
+    end
+
+    #
+    # Freeze the opts Hash so that no options can be modified
+    # once the crawl begins
+    #
+    def freeze_options
+      @opts.freeze
+      @opts.each_key { |key| @opts[key].freeze }
+      @opts[:cookies].each_key { |key| @opts[:cookies][key].freeze } rescue nil
     end
 
     # Generate Authorization string and set authorization opts
@@ -213,19 +229,19 @@ module Anemone
     # Execute the after_crawl blocks
     #
     def do_after_crawl_blocks
-      @after_crawl_blocks.each { |b| b.call(@pages) }
+      @after_crawl_blocks.each { |block| block.call(@pages) }
     end
 
     #
     # Execute the on_every_page blocks for *page*
     #
     def do_page_blocks(page)
-      @on_every_page_blocks.each do |blk|
-        blk.call(page)
+      @on_every_page_blocks.each do |block|
+        block.call(page)
       end
 
-      @on_pages_like_blocks.each do |pattern, blks|
-        blks.each { |blk| blk.call(page) } if page.url.to_s =~ pattern
+      @on_pages_like_blocks.each do |pattern, blocks|
+        blocks.each { |block| block.call(page) } if page.url.to_s =~ pattern
       end
     end
 
@@ -263,7 +279,7 @@ module Anemone
     # its URL matches a skip_link pattern.
     #
     def skip_link?(link)
-      @skip_link_patterns.any? { |p| link.path =~ p }
+      @skip_link_patterns.any? { |pattern| link.path =~ pattern }
     end
 
   end

data/lib/anemone/http.rb

@@ -1,14 +1,19 @@
 require 'net/https'
 require 'anemone/page'
+require 'anemone/cookie_store'
 
 module Anemone
   class HTTP
     # Maximum number of redirects to follow on each get_response
     REDIRECT_LIMIT = 5
 
+    # CookieStore for this HTTP client
+    attr_reader :cookie_store
+
     def initialize(opts = {})
       @connections = {}
       @opts = opts
+      @cookie_store = CookieStore.new(@opts[:cookies])
     end
 
     #
@@ -47,6 +52,28 @@ module Anemone
       end
     end
 
+    #
+    # The maximum number of redirects to follow
+    #
+    def redirect_limit
+      @opts[:redirect_limit] || REDIRECT_LIMIT
+    end
+
+    #
+    # The user-agent string which will be sent with each request,
+    # or nil if no such option is set
+    #
+    def user_agent
+      @opts[:user_agent]
+    end
+
+    #
+    # Does this HTTP client accept cookies from the server?
+    #
+    def accept_cookies?
+      @opts[:accept_cookies]
+    end
+
     private
 
     #
@@ -55,22 +82,19 @@ module Anemone
     # for each response.
     #
     def get(url, referer = nil)
-      response, response_time = get_response(url, referer)
-      code = Integer(response.code)
-      loc = url
-      redirect_to = response.is_a?(Net::HTTPRedirection) ?  URI(response['location']) : nil
-      yield response, code, loc, redirect_to, response_time
-
       limit = redirect_limit
-      while response.is_a?(Net::HTTPRedirection) and limit > 0
-          loc = redirect_to
+      loc = url
+      begin
+          # if redirected to a relative url, merge it with the host of the original
+          # request url
           loc = url.merge(loc) if loc.relative?
+
           response, response_time = get_response(loc, referer)
           code = Integer(response.code)
           redirect_to = response.is_a?(Net::HTTPRedirection) ?  URI(response['location']) : nil
           yield response, code, loc, redirect_to, response_time
           limit -= 1
-      end
+      end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
     end
 
     #
@@ -82,6 +106,7 @@ module Anemone
       opts = {}
       opts['User-Agent'] = user_agent if user_agent
       opts['Referer'] = referer.to_s if referer
+      opts['Cookie'] = @cookie_store.to_s unless @cookie_store.empty? || (!accept_cookies? && @opts[:cookies].nil?)
       opts['Authorization'] = authorization if authorization
 
       retries = 0
@@ -90,6 +115,7 @@ module Anemone
         response = connection(url).get(full_path, opts)
         finish = Time.now()
         response_time = ((finish - start) * 1000).round
+        @cookie_store.merge!(response['Set-Cookie']) if accept_cookies?
         return response, response_time
       rescue EOFError
         refresh_connection(url)
@@ -117,18 +143,17 @@ module Anemone
       @connections[url.host][url.port] = http.start
     end
 
-    def redirect_limit
-      @opts[:redirect_limit] || REDIRECT_LIMIT
-    end
-
-    def user_agent
-      @opts[:user_agent]
-    end
-
     def verbose?
       @opts[:verbose]
     end
 
+    #
+    # Allowed to connect to the requested url?
+    #
+    def allowed?(to_url, from_url)
+      to_url.host.nil? || (to_url.host == from_url.host)
+    end
+
     def authorization
       @opts[:authorization]
     end

data/lib/anemone/page.rb

@@ -1,5 +1,6 @@
 require 'nokogiri'
 require 'ostruct'
+require 'webrick/cookie'
 
 module Anemone
   class Page
@@ -94,6 +95,13 @@ module Anemone
       @fetched
     end
 
+    #
+    # Array of cookies received with this page as WEBrick::Cookie objects.
+    #
+    def cookies
+      WEBrick::Cookie.parse_set_cookies(@headers['Set-Cookie']) rescue []
+    end
+
     #
     # The content-type returned by the HTTP request for this page
     #

data/spec/cookie_store_spec.rb

@@ -0,0 +1,27 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+module Anemone
+  describe CookieStore do
+
+    it "should start out empty if no cookies are specified" do
+      CookieStore.new.empty?.should be true
+    end
+
+    it "should accept a Hash of cookies in the constructor" do
+      CookieStore.new({'test' => 'cookie'})['test'].value.should == 'cookie'
+    end
+
+    it "should be able to merge an HTTP cookie string" do
+      cs = CookieStore.new({'a' => 'a', 'b' => 'b'})
+      cs.merge! "a=A; path=/, c=C; path=/"
+      cs['a'].value.should == 'A'
+      cs['b'].value.should == 'b'
+      cs['c'].value.should == 'C'
+    end
+
+    it "should have a to_s method to turn the cookies into a string for the HTTP Cookie header" do
+      CookieStore.new({'a' => 'a', 'b' => 'b'}).to_s.should == 'a=a;b=b'
+    end
+
+  end
+end

data/spec/core_spec.rb

@@ -19,7 +19,7 @@ module Anemone
         Anemone.crawl(pages[0].url, @opts).should have(4).pages
       end
 
-      it "should not leave the original domain" do
+      it "should not follow links that leave the original domain" do
         pages = []
         pages << FakePage.new('0', :links => ['1'], :hrefs => 'http://www.other.com/')
         pages << FakePage.new('1')
@@ -30,6 +30,17 @@ module Anemone
         core.pages.keys.should_not include('http://www.other.com/')
       end
 
+      it "should not follow redirects that leave the original domain" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'], :redirect => 'http://www.other.com/')
+        pages << FakePage.new('1')
+
+        core = Anemone.crawl(pages[0].url, @opts)
+
+        core.should have(2).pages
+        core.pages.keys.should_not include('http://www.other.com/')
+      end
+
       it "should follow http redirects" do
         pages = []
         pages << FakePage.new('0', :links => ['1'])
@@ -143,6 +154,24 @@ module Anemone
         urls.should_not include(pages[1].url)
       end
 
+      it "should be able to set cookies to send with HTTP requests" do
+        cookies = {:a => '1', :b => '2'}
+        core = Anemone.crawl(FakePage.new('0').url) do |anemone|
+          anemone.cookies = cookies
+        end
+        core.opts[:cookies].should == cookies
+      end
+
+      it "should freeze the options once the crawl begins" do
+        core = Anemone.crawl(FakePage.new('0').url) do |anemone|
+          anemone.threads = 4
+          anemone.on_every_page do
+            lambda {anemone.threads = 2}.should raise_error
+          end
+        end
+        core.opts[:threads].should == 4
+      end
+
       describe "many pages" do
         before(:each) do
           @pages, size = [], 5

data/spec/fakeweb_helper.rb

@@ -9,12 +9,12 @@ FakeWeb.allow_net_connect = false
 
 module Anemone
   SPEC_DOMAIN = "http://www.example.com/"
-  
+
   class FakePage
     attr_accessor :links
     attr_accessor :hrefs
     attr_accessor :body
-    
+
     def initialize(name = '', options = {})
       @name = name
       @links = [options[:links]].flatten if options.has_key?(:links)
@@ -22,30 +22,38 @@ module Anemone
       @redirect = options[:redirect] if options.has_key?(:redirect)
       @content_type = options[:content_type] || "text/html"
       @body = options[:body]
-      
+
       create_body unless @body
       add_to_fakeweb
     end
-    
+
     def url
       SPEC_DOMAIN + @name
     end
-    
+
     private
-    
+
     def create_body
       @body = "<html><body>"
       @links.each{|l| @body += "<a href=\"#{SPEC_DOMAIN}#{l}\"></a>"} if @links
       @hrefs.each{|h| @body += "<a href=\"#{h}\"></a>"} if @hrefs
       @body += "</body></html>"
     end
-    
+
     def add_to_fakeweb
       options = {:body => @body, :content_type => @content_type, :status => [200, "OK"]}
-      
+
       if @redirect
-        options[:status] = [301, "Permanently Moved"] 
-        options[:location] = SPEC_DOMAIN + @redirect
+        options[:status] = [301, "Permanently Moved"]
+
+        # only prepend SPEC_DOMAIN if a relative url (without an http scheme) was specified
+        redirect_url = (@redirect =~ /http/) ? @redirect : SPEC_DOMAIN + @redirect
+        options[:location] = redirect_url
+
+        # register the page this one redirects to
+        FakeWeb.register_uri(:get, redirect_url, {:body => '',
+                                                  :content_type => @content_type,
+                                                  :status => [200, "OK"]})
       end
 
       FakeWeb.register_uri(:get, SPEC_DOMAIN + @name, options)

data/spec/http_spec.rb

@@ -15,9 +15,10 @@ module Anemone
           end
         end
 
-        http = HTTP.new
+        http = Anemone::HTTP.new
         http.fetch_page(SPEC_DOMAIN).should be_an_instance_of(Page)
       end
+
     end
   end
-end
+end

data/spec/page_spec.rb

@@ -68,5 +68,10 @@ module Anemone
       @page.should respond_to(:response_time)
     end
 
+    it "should have the cookies received with the page" do
+      @page.should respond_to(:cookies)
+      @page.cookies.should == []
+    end
+
   end
 end

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: spk-anemone
 version: !ruby/object:Gem::Version 
-  version: 0.3.1
+  hash: 15
+  prerelease: false
+  segments: 
+  - 0
+  - 4
+  - 0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Chris Kite
@@ -9,29 +15,41 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-29 00:00:00 +01:00
+date: 2010-08-18 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 1
+        - 4
+        - 1
         version: 1.4.1
-    version: 
+  type: :runtime
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: robots
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 7
+        - 2
         version: 0.7.2
-    version: 
+  type: :runtime
+  version_requirements: *id002
 description: 
 email: 
 executables: 
@@ -46,6 +64,7 @@ files:
 - README.rdoc
 - bin/anemone
 - lib/anemone.rb
+- lib/anemone/cookie_store.rb
 - lib/anemone/core.rb
 - lib/anemone/http.rb
 - lib/anemone/page.rb
@@ -60,6 +79,15 @@ files:
 - lib/anemone/cli/count.rb
 - lib/anemone/cli/pagedepth.rb
 - lib/anemone/cli/serialize.rb
+- spec/anemone_spec.rb
+- spec/cookie_store_spec.rb
+- spec/core_spec.rb
+- spec/page_spec.rb
+- spec/page_store_spec.rb
+- spec/http_spec.rb
+- spec/storage_spec.rb
+- spec/fakeweb_helper.rb
+- spec/spec_helper.rb
 has_rdoc: true
 homepage: http://anemone.rubyforge.org
 licenses: []
@@ -73,26 +101,33 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: anemone
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Anemone web-spider framework
 test_files: 
 - spec/anemone_spec.rb
+- spec/cookie_store_spec.rb
 - spec/core_spec.rb
 - spec/page_spec.rb
 - spec/page_store_spec.rb