anemone 0.3.2 → 0.4.0

data/CHANGELOG.rdoc

@@ -1,3 +1,9 @@
+== 0.4.0 / 2010-04-08
+
+* Major enchancements
+
+  * Cookies can be accepted and sent with each HTTP request.
+
 == 0.3.2 / 2010-02-04
 
 * Bug fixes

data/lib/anemone/cookie_store.rb

@@ -0,0 +1,35 @@
+require 'delegate'
+require 'webrick/cookie'
+
+class WEBrick::Cookie
+  def expired?
+    !!expires && expires < Time.now
+  end
+end
+
+module Anemone
+  class CookieStore < DelegateClass(Hash)
+
+    def initialize(cookies = nil)
+      @cookies = {}
+      cookies.each { |name, value| @cookies[name] = WEBrick::Cookie.new(name, value) } if cookies
+      super(@cookies)
+    end
+
+    def merge!(set_cookie_str)
+      begin
+        cookie_hash = WEBrick::Cookie.parse_set_cookies(set_cookie_str).inject({}) do |hash, cookie|
+          hash[cookie.name] = cookie if !!cookie
+          hash
+        end
+        @cookies.merge! cookie_hash
+      rescue
+      end
+    end
+
+    def to_s
+      @cookies.values.reject { |cookie| cookie.expired? }.map { |cookie| "#{cookie.name}=#{cookie.value}" }.join(';')
+    end
+
+  end
+end

data/lib/anemone/core.rb

@@ -7,7 +7,7 @@ require 'anemone/storage'
 
 module Anemone
 
-  VERSION = '0.3.2';
+  VERSION = '0.4.0';
 
   #
   # Convenience method to start a crawl
@@ -41,13 +41,17 @@ module Anemone
       # number of times HTTP redirects will be followed
       :redirect_limit => 5,
       # storage engine defaults to Hash in +process_options+ if none specified
-      :storage => nil
+      :storage => nil,
+      # Hash of cookie name => value to send with HTTP requests
+      :cookies => nil,
+      # accept cookies from the server and send them back?
+      :accept_cookies => false
     }
 
     # Create setter methods for all options to be called from the crawl block
     DEFAULT_OPTS.keys.each do |key|
-      define_method "#{key}=" do |*args|
-        @opts[key.to_sym] = *args
+      define_method "#{key}=" do |value|
+        @opts[key.to_sym] = value
       end
     end
 
@@ -173,7 +177,7 @@ module Anemone
         end
       end
 
-      @tentacles.each { |t| t.join }
+      @tentacles.each { |thread| thread.join }
       do_after_crawl_blocks
       self
     end
@@ -185,25 +189,37 @@ module Anemone
       @opts[:threads] = 1 if @opts[:delay] > 0
       @pages = PageStore.new(@opts[:storage] || Anemone::Storage.Hash)
       @robots = Robots.new(@opts[:user_agent]) if @opts[:obey_robots_txt]
+
+      freeze_options
+    end
+
+    #
+    # Freeze the opts Hash so that no options can be modified
+    # once the crawl begins
+    #
+    def freeze_options
+      @opts.freeze
+      @opts.each_key { |key| @opts[key].freeze }
+      @opts[:cookies].each_key { |key| @opts[:cookies][key].freeze } rescue nil
     end
 
     #
     # Execute the after_crawl blocks
     #
     def do_after_crawl_blocks
-      @after_crawl_blocks.each { |b| b.call(@pages) }
+      @after_crawl_blocks.each { |block| block.call(@pages) }
     end
 
     #
     # Execute the on_every_page blocks for *page*
     #
     def do_page_blocks(page)
-      @on_every_page_blocks.each do |blk|
-        blk.call(page)
+      @on_every_page_blocks.each do |block|
+        block.call(page)
       end
 
-      @on_pages_like_blocks.each do |pattern, blks|
-        blks.each { |blk| blk.call(page) } if page.url.to_s =~ pattern
+      @on_pages_like_blocks.each do |pattern, blocks|
+        blocks.each { |block| block.call(page) } if page.url.to_s =~ pattern
       end
     end
 
@@ -241,7 +257,7 @@ module Anemone
     # its URL matches a skip_link pattern.
     #
     def skip_link?(link)
-      @skip_link_patterns.any? { |p| link.path =~ p }
+      @skip_link_patterns.any? { |pattern| link.path =~ pattern }
     end
 
   end

data/lib/anemone/http.rb

@@ -1,14 +1,19 @@
 require 'net/https'
 require 'anemone/page'
+require 'anemone/cookie_store'
 
 module Anemone
   class HTTP
     # Maximum number of redirects to follow on each get_response
     REDIRECT_LIMIT = 5
 
+    # CookieStore for this HTTP client
+    attr_reader :cookie_store
+
     def initialize(opts = {})
       @connections = {}
       @opts = opts
+      @cookie_store = CookieStore.new(@opts[:cookies])
     end
 
     #
@@ -47,6 +52,28 @@ module Anemone
       end
     end
 
+    #
+    # The maximum number of redirects to follow
+    #
+    def redirect_limit
+      @opts[:redirect_limit] || REDIRECT_LIMIT
+    end
+
+    #
+    # The user-agent string which will be sent with each request,
+    # or nil if no such option is set
+    #
+    def user_agent
+      @opts[:user_agent]
+    end
+
+    #
+    # Does this HTTP client accept cookies from the server?
+    #
+    def accept_cookies?
+      @opts[:accept_cookies]
+    end
+
     private
 
     #
@@ -55,22 +82,19 @@ module Anemone
     # for each response.
     #
     def get(url, referer = nil)
-      response, response_time = get_response(url, referer)
-      code = Integer(response.code)
-      loc = url
-      redirect_to = response.is_a?(Net::HTTPRedirection) ?  URI(response['location']) : nil
-      yield response, code, loc, redirect_to, response_time
-
       limit = redirect_limit
-      while redirect_to && allowed?(redirect_to, url) && limit > 0
-          loc = redirect_to
+      loc = url
+      begin
+          # if redirected to a relative url, merge it with the host of the original
+          # request url
           loc = url.merge(loc) if loc.relative?
+
           response, response_time = get_response(loc, referer)
           code = Integer(response.code)
           redirect_to = response.is_a?(Net::HTTPRedirection) ?  URI(response['location']) : nil
           yield response, code, loc, redirect_to, response_time
           limit -= 1
-      end
+      end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
     end
 
     #
@@ -82,6 +106,7 @@ module Anemone
       opts = {}
       opts['User-Agent'] = user_agent if user_agent
       opts['Referer'] = referer.to_s if referer
+      opts['Cookie'] = @cookie_store.to_s unless @cookie_store.empty? || (!accept_cookies? && @opts[:cookies].nil?)
 
       retries = 0
       begin
@@ -89,6 +114,7 @@ module Anemone
         response = connection(url).get(full_path, opts)
         finish = Time.now()
         response_time = ((finish - start) * 1000).round
+        @cookie_store.merge!(response['Set-Cookie']) if accept_cookies?
         return response, response_time
       rescue EOFError
         refresh_connection(url)
@@ -116,14 +142,6 @@ module Anemone
       @connections[url.host][url.port] = http.start
     end
 
-    def redirect_limit
-      @opts[:redirect_limit] || REDIRECT_LIMIT
-    end
-
-    def user_agent
-      @opts[:user_agent]
-    end
-
     def verbose?
       @opts[:verbose]
     end

data/lib/anemone/page.rb

@@ -1,5 +1,6 @@
 require 'nokogiri'
 require 'ostruct'
+require 'webrick/cookie'
 
 module Anemone
   class Page
@@ -92,6 +93,13 @@ module Anemone
       @fetched
     end
 
+    #
+    # Array of cookies received with this page as WEBrick::Cookie objects.
+    #
+    def cookies
+      WEBrick::Cookie.parse_set_cookies(@headers['Set-Cookie']) rescue []
+    end
+
     #
     # The content-type returned by the HTTP request for this page
     #

data/spec/cookie_store_spec.rb

@@ -0,0 +1,27 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+module Anemone
+  describe CookieStore do
+
+    it "should start out empty if no cookies are specified" do
+      CookieStore.new.empty?.should be true
+    end
+
+    it "should accept a Hash of cookies in the constructor" do
+      CookieStore.new({'test' => 'cookie'})['test'].value.should == 'cookie'
+    end
+
+    it "should be able to merge an HTTP cookie string" do
+      cs = CookieStore.new({'a' => 'a', 'b' => 'b'})
+      cs.merge! "a=A; path=/, c=C; path=/"
+      cs['a'].value.should == 'A'
+      cs['b'].value.should == 'b'
+      cs['c'].value.should == 'C'
+    end
+
+    it "should have a to_s method to turn the cookies into a string for the HTTP Cookie header" do
+      CookieStore.new({'a' => 'a', 'b' => 'b'}).to_s.should == 'a=a;b=b'
+    end
+
+  end
+end

data/spec/core_spec.rb

@@ -154,6 +154,24 @@ module Anemone
         urls.should_not include(pages[1].url)
       end
 
+      it "should be able to set cookies to send with HTTP requests" do
+        cookies = {:a => '1', :b => '2'}
+        core = Anemone.crawl(FakePage.new('0').url) do |anemone|
+          anemone.cookies = cookies
+        end
+        core.opts[:cookies].should == cookies
+      end
+
+      it "should freeze the options once the crawl begins" do
+        core = Anemone.crawl(FakePage.new('0').url) do |anemone|
+          anemone.threads = 4
+          anemone.on_every_page do
+            lambda {anemone.threads = 2}.should raise_error
+          end
+        end
+        core.opts[:threads].should == 4
+      end
+
       describe "many pages" do
         before(:each) do
           @pages, size = [], 5

data/spec/http_spec.rb

@@ -15,9 +15,10 @@ module Anemone
           end
         end
 
-        http = HTTP.new
+        http = Anemone::HTTP.new
         http.fetch_page(SPEC_DOMAIN).should be_an_instance_of(Page)
       end
+
     end
   end
-end
+end

data/spec/page_spec.rb

@@ -68,5 +68,10 @@ module Anemone
       @page.should respond_to(:response_time)
     end
 
+    it "should have the cookies received with the page" do
+      @page.should respond_to(:cookies)
+      @page.cookies.should == []
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: anemone
 version: !ruby/object:Gem::Version 
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors: 
 - Chris Kite
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-04 00:00:00 -06:00
+date: 2010-04-08 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -46,6 +46,7 @@ files:
 - README.rdoc
 - bin/anemone
 - lib/anemone.rb
+- lib/anemone/cookie_store.rb
 - lib/anemone/core.rb
 - lib/anemone/http.rb
 - lib/anemone/page.rb
@@ -93,6 +94,7 @@ specification_version: 3
 summary: Anemone web-spider framework
 test_files: 
 - spec/anemone_spec.rb
+- spec/cookie_store_spec.rb
 - spec/core_spec.rb
 - spec/page_spec.rb
 - spec/page_store_spec.rb