anemone 0.4.0 → 0.5.0

data/CHANGELOG.rdoc

@@ -1,3 +1,19 @@
+== 0.5.0 / 2010-09-01
+
+* Major enhancements
+
+  * Added page storage engines for MongoDB and Redis
+
+* Minor enhancements
+
+  * Use xpath for link parsing instead of CSS (faster) (Marc Seeger)
+  * Added skip_query_strings option to skip links with query strings (Joost Baaij)
+
+* Bug fixes
+
+  * Only consider status code 300..307 a redirect (Marc Seeger)
+  * Canonicalize redirect links (Marc Seeger)
+
 == 0.4.0 / 2010-04-08
 
 * Major enchancements

data/README.rdoc

@@ -8,7 +8,7 @@ See http://anemone.rubyforge.org for more information.
 
 == Features
 * Multi-threaded design for high performance
-* Tracks 301 HTTP redirects to understand a page's aliases
+* Tracks 301 HTTP redirects
 * Built-in BFS algorithm for determining page depth
 * Allows exclusion of URLs based on regular expressions
 * Choose the links to follow on each page with focus_crawl()
@@ -16,7 +16,7 @@ See http://anemone.rubyforge.org for more information.
 * Records response time for each page
 * CLI program can list all pages in a domain, calculate page depths, and more
 * Obey robots.txt
-* In-memory or persistent storage of pages during crawl, using TokyoCabinet or PStore
+* In-memory or persistent storage of pages during crawl, using TokyoCabinet, MongoDB, or Redis
 
 == Examples
 See the scripts under the <tt>lib/anemone/cli</tt> directory for examples of several useful Anemone tasks.
@@ -24,3 +24,13 @@ See the scripts under the <tt>lib/anemone/cli</tt> directory for examples of sev
 == Requirements
 * nokogiri
 * robots
+
+== Development
+To test and develop this gem, additional requirements are:
+* rspec
+* fakeweb
+* tokyocabinet
+* mongo
+* redis
+
+You will need to have {Tokyo Cabinet}[http://fallabs.com/tokyocabinet/], {MongoDB}[http://www.mongodb.org/], and {Redis}[http://code.google.com/p/redis/] installed on your system and running.

data/Rakefile

@@ -0,0 +1,26 @@
+require 'rubygems'
+require 'rake'
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "anemone #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.5.0

data/lib/anemone/core.rb

@@ -2,12 +2,14 @@ require 'thread'
 require 'robots'
 require 'anemone/tentacle'
 require 'anemone/page'
+require 'anemone/exceptions'
 require 'anemone/page_store'
 require 'anemone/storage'
+require 'anemone/storage/base'
 
 module Anemone
 
-  VERSION = '0.4.0';
+  VERSION = '0.5.0';
 
   #
   # Convenience method to start a crawl
@@ -45,7 +47,9 @@ module Anemone
       # Hash of cookie name => value to send with HTTP requests
       :cookies => nil,
       # accept cookies from the server and send them back?
-      :accept_cookies => false
+      :accept_cookies => false,
+      # skip any link with a query string? e.g. http://foo.com/?u=user
+      :skip_query_strings => false
     }
 
     # Create setter methods for all options to be called from the crawl block
@@ -187,7 +191,8 @@ module Anemone
     def process_options
       @opts = DEFAULT_OPTS.merge @opts
       @opts[:threads] = 1 if @opts[:delay] > 0
-      @pages = PageStore.new(@opts[:storage] || Anemone::Storage.Hash)
+      storage = Anemone::Storage::Base.new(@opts[:storage] || Anemone::Storage.Hash)
+      @pages = PageStore.new(storage)
       @robots = Robots.new(@opts[:user_agent]) if @opts[:obey_robots_txt]
 
       freeze_options
@@ -241,15 +246,40 @@ module Anemone
     # Returns +false+ otherwise.
     #
     def visit_link?(link, from_page = nil)
-      allowed = @opts[:obey_robots_txt] ? @robots.allowed?(link) : true
+      !@pages.has_page?(link) &&
+      !skip_link?(link) &&
+      !skip_query_string?(link) &&
+      allowed(link) &&
+      !too_deep?(from_page)
+    end
+
+    #
+    # Returns +true+ if we are obeying robots.txt and the link
+    # is granted access in it. Always returns +true+ when we are
+    # not obeying robots.txt.
+    #
+    def allowed(link)
+      @opts[:obey_robots_txt] ? @robots.allowed?(link) : true
+    end
 
+    #
+    # Returns +true+ if we are over the page depth limit.
+    # This only works when coming from a page and with the +depth_limit+ option set.
+    # When neither is the case, will always return +false+.
+    def too_deep?(from_page)
       if from_page && @opts[:depth_limit]
-        too_deep = from_page.depth >= @opts[:depth_limit]
+        from_page.depth >= @opts[:depth_limit]
       else
-        too_deep = false
+        false
       end
-
-      !@pages.has_page?(link) && !skip_link?(link) && allowed && !too_deep
+    end
+    
+    #
+    # Returns +true+ if *link* should not be visited because
+    # it has a query string and +skip_query_strings+ is true.
+    #
+    def skip_query_string?(link)
+      @opts[:skip_query_strings] && link.query
     end
 
     #

data/lib/anemone/exceptions.rb

@@ -0,0 +1,5 @@
+module Anemone
+  class Error < ::StandardError
+    attr_accessor :wrapped_exception
+  end
+end

data/lib/anemone/http.rb

@@ -91,7 +91,7 @@ module Anemone
 
           response, response_time = get_response(loc, referer)
           code = Integer(response.code)
-          redirect_to = response.is_a?(Net::HTTPRedirection) ?  URI(response['location']) : nil
+          redirect_to = response.is_a?(Net::HTTPRedirection) ?  URI(response['location']).normalize : nil
           yield response, code, loc, redirect_to, response_time
           limit -= 1
       end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0

data/lib/anemone/page.rb

@@ -59,8 +59,8 @@ module Anemone
       @links = []
       return @links if !doc
 
-      doc.css('a').each do |a|
-        u = a.attributes['href'].content rescue nil
+      doc.search("//a[@href]").each do |a|
+        u = a['href']
         next if u.nil? or u.empty?
         abs = to_absolute(URI(u)) rescue next
         @links << abs if in_domain?(abs)
@@ -120,7 +120,7 @@ module Anemone
     # otherwise.
     #
     def redirect?
-      (300..399).include?(@code)
+      (300..307).include?(@code)
     end
 
     #
@@ -165,5 +165,38 @@ module Anemone
       @url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched = ary
     end
 
+    def to_hash
+      {'url' => @url.to_s,
+       'headers' => Marshal.dump(@headers),
+       'data' => Marshal.dump(@data),
+       'body' => @body,
+       'links' => links.map(&:to_s), 
+       'code' => @code,
+       'visited' => @visited,
+       'depth' => @depth,
+       'referer' => @referer.to_s,
+       'redirect_to' => @redirect_to.to_s,
+       'response_time' => @response_time,
+       'fetched' => @fetched}
+    end
+
+    def self.from_hash(hash)
+      page = self.new(URI(hash['url']))
+      {'@headers' => Marshal.load(hash['headers']),
+       '@data' => Marshal.load(hash['data']),
+       '@body' => hash['body'],
+       '@links' => hash['links'].map { |link| URI(link) },
+       '@code' => hash['code'].to_i,
+       '@visited' => hash['visited'],
+       '@depth' => hash['depth'].to_i,
+       '@referer' => hash['referer'],
+       '@redirect_to' => URI(hash['redirect_to']),
+       '@response_time' => hash['response_time'].to_i,
+       '@fetched' => hash['fetched']
+      }.each do |var, value|
+        page.instance_variable_set(var, value)
+      end
+      page
+    end
   end
 end

data/lib/anemone/storage.rb

@@ -2,7 +2,10 @@ module Anemone
   module Storage
 
     def self.Hash(*args)
-      Hash.new(*args)
+      hash = Hash.new(*args)
+      # add close method for compatibility with Storage::Base
+      class << hash; def close; end; end
+      hash
     end
 
     def self.PStore(*args)
@@ -10,10 +13,22 @@ module Anemone
       self::PStore.new(*args)
     end
 
-    def self.TokyoCabinet(file)
+    def self.TokyoCabinet(file = 'anemone.tch')
       require 'anemone/storage/tokyo_cabinet'
       self::TokyoCabinet.new(file)
     end
 
+    def self.MongoDB(mongo_db = nil, collection_name = 'pages')
+      require 'anemone/storage/mongodb'
+      mongo_db ||= Mongo::Connection.new.db('anemone')
+      raise "First argument must be an instance of Mongo::DB" unless mongo_db.is_a?(Mongo::DB)
+      self::MongoDB.new(mongo_db, collection_name)
+    end
+
+    def self.Redis(opts = {})
+      require 'anemone/storage/redis'
+      self::Redis.new(opts)
+    end
+
   end
 end

data/lib/anemone/storage/base.rb

@@ -0,0 +1,75 @@
+require 'anemone/storage/exceptions'
+
+module Anemone
+  module Storage
+    class Base
+
+      def initialize(adapter)
+        @adap = adapter
+
+        # verify adapter conforms to this class's methods
+        methods.each do |method|
+          if !@adap.respond_to?(method.to_sym)
+            raise "Storage adapter must support method #{method}"
+          end
+        end
+      end
+
+      def [](key)
+        @adap[key]
+        rescue
+          puts key
+          raise RetrievalError, $! 
+      end
+
+      def []=(key, value)
+        @adap[key] = value
+        rescue
+          raise InsertionError, $!
+      end
+
+      def delete(key)
+        @adap.delete(key)
+        rescue
+          raise DeletionError, $!
+      end
+
+      def each
+        @adap.each { |k, v| yield k, v }
+        rescue
+          raise GenericError, $!
+      end
+
+      def merge!(hash)
+        @adap.merge!(hash)
+        rescue
+          raise GenericError, $!
+      end
+
+      def close
+        @adap.close
+        rescue
+          raise CloseError, $!
+      end
+
+      def size
+        @adap.size
+        rescue
+          raise GenericError, $!
+      end
+
+      def keys
+        @adap.keys
+        rescue
+          raise GenericError, $!
+      end
+
+      def has_key?(key)
+        @adap.has_key?(key)
+        rescue
+          raise GenericError, $!
+      end
+
+    end
+  end
+end

data/lib/anemone/storage/exceptions.rb

@@ -0,0 +1,15 @@
+module Anemone
+  module Storage
+
+    class GenericError < Error; end;
+
+    class ConnectionError < Error; end
+
+    class RetrievalError < Error; end
+
+    class InsertionError < Error; end
+
+    class CloseError < Error; end
+
+  end
+end

data/lib/anemone/storage/mongodb.rb

@@ -0,0 +1,89 @@
+begin
+  require 'mongo'
+rescue LoadError
+  puts "You need the mongo gem to use Anemone::Storage::MongoDB"
+  exit
+end
+
+module Anemone
+  module Storage
+    class MongoDB 
+
+      BINARY_FIELDS = %w(body headers data)
+
+      def initialize(mongo_db, collection_name)
+        @db = mongo_db
+        @collection = @db[collection_name]
+        @collection.remove
+        @collection.create_index 'url'
+      end
+
+      def [](url)
+        if value = @collection.find_one('url' => url.to_s)
+          load_page(value)
+        end
+      end
+
+      def []=(url, page)
+        hash = page.to_hash
+        BINARY_FIELDS.each do |field|
+          hash[field] = BSON::Binary.new(hash[field]) unless hash[field].nil?
+        end
+        @collection.update(
+          {'url' => page.url.to_s},
+          hash,
+          :upsert => true
+        )
+      end
+
+      def delete(url)
+        page = self[url]
+        @collection.remove('url' => url.to_s)
+        page
+      end
+
+      def each
+        @collection.find do |cursor|
+          cursor.each do |doc|
+            page = load_page(doc)
+            yield page.url.to_s, page 
+          end
+        end
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      def size
+        @collection.count
+      end
+
+      def keys
+        keys = []
+        self.each { |k, v| keys << k.to_s }
+        keys
+      end
+
+      def has_key?(url)
+        !!@collection.find_one('url' => url.to_s)
+      end
+
+      def close
+        @db.connection.close
+      end
+
+      private
+
+      def load_page(hash)
+        BINARY_FIELDS.each do |field|
+          hash[field] = hash[field].to_s
+        end
+        Page.from_hash(hash)
+      end
+
+    end
+  end
+end
+

data/lib/anemone/storage/pstore.rb

@@ -43,6 +43,8 @@ module Anemone
         self
       end
 
+      def close; end
+
     end
   end
 end

data/lib/anemone/storage/redis.rb

@@ -0,0 +1,90 @@
+require 'redis'
+
+module Anemone
+  module Storage
+    class Redis
+
+      MARSHAL_FIELDS = %w(links visited fetched)
+
+      def initialize(opts = {})
+        @redis = ::Redis.new(opts)
+        @key_prefix = opts[:key_prefix] || 'anemone'
+        keys.each { |key| delete(key) }
+      end
+
+      def [](key)
+        rkey = "#{@key_prefix}:pages:#{key.to_s}"
+        rget(rkey)
+      end
+
+      def []=(key, value)
+        rkey = "#{@key_prefix}:pages:#{key.to_s}"
+        hash = value.to_hash
+        MARSHAL_FIELDS.each do |field|
+          hash[field] = Marshal.dump(hash[field])
+        end
+        hash.each do |field, value|
+          @redis.hset(rkey, field, value)
+        end
+      end
+
+      def delete(key)
+        rkey = "#{@key_prefix}:pages:#{key.to_s}"
+        page = self[key]
+        @redis.del(rkey)
+        page
+      end
+
+      def each
+        rkeys = @redis.keys("#{@key_prefix}:pages:*")
+        rkeys.each do |rkey|
+          page = rget(rkey)
+          yield page.url.to_s, page
+        end
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      def size
+        @redis.keys("#{@key_prefix}:pages:*").size
+      end
+
+      def keys
+        keys = []
+        self.each { |k, v| keys << k.to_s }
+        keys
+      end
+
+      def has_key?(key)
+        rkey = "#{@key_prefix}:pages:#{key.to_s}"
+        @redis.exists(rkey)
+      end
+
+      def close
+        @redis.quit
+      end
+
+      private
+
+      def load_value(hash)
+        MARSHAL_FIELDS.each do |field|
+          unless hash[field].nil? || hash[field] == ''
+            hash[field] = Marshal.load(hash[field]) 
+          end
+        end
+        Page.from_hash(hash)
+      end
+
+      def rget(rkey)
+        hash = @redis.hgetall(rkey)
+        if !!hash
+          load_value(hash)
+        end
+      end
+
+    end
+  end
+end

data/spec/core_spec.rb

@@ -72,6 +72,19 @@ module Anemone
         core.pages.keys.should_not include(pages[2].url)
       end
 
+      it "should be able to skip links with query strings" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1?foo=1', '2'])
+        pages << FakePage.new('1?foo=1')
+        pages << FakePage.new('2')
+        
+        core = Anemone.crawl(pages[0].url, @opts) do |a|
+          a.skip_query_strings = true
+        end
+        
+        core.should have(2).pages
+      end
+
       it "should be able to skip links based on a RegEx" do
         pages = []
         pages << FakePage.new('0', :links => ['1', '2'])

data/spec/page_spec.rb

@@ -6,7 +6,7 @@ module Anemone
     before(:each) do
       FakeWeb.clean_registry
       @http = Anemone::HTTP.new
-      @page = @http.fetch_page(FakePage.new('home').url)
+      @page = @http.fetch_page(FakePage.new('home', :links => '1').url)
     end
 
     it "should indicate whether it successfully fetched via HTTP" do
@@ -73,5 +73,20 @@ module Anemone
       @page.cookies.should == []
     end
 
+    it "should have a to_hash method that converts the page to a hash" do
+      hash = @page.to_hash
+      hash['url'].should == @page.url.to_s
+      hash['referer'].should == @page.referer.to_s
+      hash['links'].should == @page.links.map(&:to_s)
+    end
+
+    it "should have a from_hash method to convert from a hash to a Page" do
+      page = @page.dup
+      page.depth = 1
+      converted = Page.from_hash(page.to_hash)
+      converted.links.should == page.links
+      converted.depth.should == page.depth
+    end
+
   end
 end

data/spec/page_store_spec.rb

@@ -1,5 +1,5 @@
 require File.dirname(__FILE__) + '/spec_helper'
-%w[pstore tokyo_cabinet].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
 
 module Anemone
   describe PageStore do
@@ -9,7 +9,7 @@ module Anemone
     end
 
     shared_examples_for "page storage" do
-      it "should be able to computer single-source shortest paths in-place" do
+      it "should be able to compute single-source shortest paths in-place" do
         pages = []
         pages << FakePage.new('0', :links => ['1', '3'])
         pages << FakePage.new('1', :redirect => '2')
@@ -124,5 +124,29 @@ module Anemone
       end
     end
 
+    describe Storage::MongoDB do
+      it_should_behave_like "page storage"
+
+      before(:each) do
+        @opts = {:storage => @store = Storage.MongoDB}
+      end
+
+      after(:each) do
+        @store.close
+      end
+    end
+
+    describe Storage::Redis do
+      it_should_behave_like "page storage"
+
+      before(:each) do
+        @opts = {:storage => @store = Storage.Redis}
+      end
+
+      after(:each) do
+        @store.close
+      end
+    end
+
   end
 end

data/spec/storage_spec.rb

@@ -1,5 +1,5 @@
 require File.dirname(__FILE__) + '/spec_helper'
-%w[pstore tokyo_cabinet].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
 
 module Anemone
   describe Storage do
@@ -23,21 +23,41 @@ module Anemone
       store.close
     end
 
+    it "should have a class method to produce a MongoDB" do
+      Anemone::Storage.should respond_to(:MongoDB)
+      store = Anemone::Storage.MongoDB
+      store.should be_an_instance_of(Anemone::Storage::MongoDB)
+      store.close
+    end
+
+    it "should have a class method to produce a Redis" do
+      Anemone::Storage.should respond_to(:Redis)
+      store = Anemone::Storage.Redis
+      store.should be_an_instance_of(Anemone::Storage::Redis)
+      store.close
+    end
+
     module Storage
       shared_examples_for "storage engine" do
+
+        before(:each) do
+          @url = SPEC_DOMAIN
+          @page = Page.new(URI(@url))
+        end
+
         it "should implement [] and []=" do
           @store.should respond_to(:[])
           @store.should respond_to(:[]=)
 
-          @store['index'] = 'test'
-          @store['index'].should == 'test'
+          @store[@url] = @page 
+          @store[@url].url.should == URI(@url)
         end
 
         it "should implement has_key?" do
           @store.should respond_to(:has_key?)
 
-          @store['index'] = 'test'
-          @store.has_key?('index').should == true
+          @store[@url] = @page
+          @store.has_key?(@url).should == true
 
           @store.has_key?('missing').should == false
         end
@@ -45,37 +65,41 @@ module Anemone
         it "should implement delete" do
           @store.should respond_to(:delete)
 
-          @store['index'] = 'test'
-          @store.delete('index').should == 'test'
-          @store.has_key?('index').should  == false
+          @store[@url] = @page
+          @store.delete(@url).url.should == @page.url
+          @store.has_key?(@url).should  == false
         end
 
         it "should implement keys" do
           @store.should respond_to(:keys)
 
-          keys = ['a', 'b', 'c']
-          keys.each { |key| @store[key] = key }
+          urls = [SPEC_DOMAIN, SPEC_DOMAIN + 'test', SPEC_DOMAIN + 'another']
+          pages = urls.map { |url| Page.new(URI(url)) }
+          urls.zip(pages).each { |arr| @store[arr[0]] = arr[1] }
 
-          @store.keys.should == keys
+          (@store.keys - urls).should == [] 
         end
 
         it "should implement each" do
           @store.should respond_to(:each)
 
-          keys = ['a', 'b', 'c']
-          keys.each { |key| @store[key] = key }
+          urls = [SPEC_DOMAIN, SPEC_DOMAIN + 'test', SPEC_DOMAIN + 'another']
+          pages = urls.map { |url| Page.new(URI(url)) }
+          urls.zip(pages).each { |arr| @store[arr[0]] = arr[1] }
 
           result = {}
           @store.each { |k, v| result[k] = v }
-          result.values.should == keys
+          (result.keys - urls).should == [] 
+          (result.values.map { |page| page.url.to_s } - urls).should == []
         end
 
         it "should implement merge!, and return self" do
           @store.should respond_to(:merge!)
 
-          hash = {'a' => 'a', 'b' => 'b', 'c' => 'c'}
+          hash = {SPEC_DOMAIN => Page.new(URI(SPEC_DOMAIN)),
+                  SPEC_DOMAIN + 'test' => Page.new(URI(SPEC_DOMAIN + 'test'))}
           merged = @store.merge! hash
-          hash.each { |key, value| @store[key].should == value }
+          hash.each { |key, value| @store[key].url.to_s.should == key }
 
           merged.should === @store
         end
@@ -115,7 +139,30 @@ module Anemone
         it "should raise an error if supplied with a file extension other than .tch" do
           lambda { Anemone::Storage.TokyoCabinet('test.tmp') }.should raise_error(RuntimeError)
         end
+      end
 
+      describe Storage::MongoDB do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @store = Storage.MongoDB
+        end
+
+        after(:each) do
+          @store.close
+        end
+      end
+
+      describe Storage::Redis do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @store = Storage.Redis
+        end
+
+        after(:each) do
+          @store.close
+        end
       end
 
     end

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: anemone
 version: !ruby/object:Gem::Version 
-  version: 0.4.0
+  hash: 11
+  prerelease: false
+  segments: 
+  - 0
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Chris Kite
@@ -9,29 +15,41 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-08 00:00:00 -05:00
+date: 2010-09-01 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 1
+        - 3
+        - 0
         version: 1.3.0
-    version: 
+  type: :runtime
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: robots
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 7
+        - 2
         version: 0.7.2
-    version: 
+  type: :runtime
+  version_requirements: *id002
 description: 
 email: 
 executables: 
@@ -41,26 +59,42 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
+- VERSION
 - LICENSE.txt
 - CHANGELOG.rdoc
 - README.rdoc
-- bin/anemone
-- lib/anemone.rb
-- lib/anemone/cookie_store.rb
-- lib/anemone/core.rb
-- lib/anemone/http.rb
-- lib/anemone/page.rb
-- lib/anemone/page_store.rb
-- lib/anemone/tentacle.rb
-- lib/anemone/storage.rb
+- Rakefile
 - lib/anemone/storage/pstore.rb
+- lib/anemone/storage/mongodb.rb
 - lib/anemone/storage/tokyo_cabinet.rb
+- lib/anemone/storage/exceptions.rb
+- lib/anemone/storage/redis.rb
+- lib/anemone/storage/base.rb
+- lib/anemone/page_store.rb
+- lib/anemone/storage.rb
+- lib/anemone/tentacle.rb
+- lib/anemone/http.rb
 - lib/anemone/cli.rb
+- lib/anemone/page.rb
+- lib/anemone/exceptions.rb
+- lib/anemone/core.rb
 - lib/anemone/cli/url_list.rb
-- lib/anemone/cli/cron.rb
+- lib/anemone/cli/serialize.rb
 - lib/anemone/cli/count.rb
+- lib/anemone/cli/cron.rb
 - lib/anemone/cli/pagedepth.rb
-- lib/anemone/cli/serialize.rb
+- lib/anemone/cookie_store.rb
+- lib/anemone.rb
+- spec/fakeweb_helper.rb
+- spec/page_spec.rb
+- spec/anemone_spec.rb
+- spec/core_spec.rb
+- spec/storage_spec.rb
+- spec/page_store_spec.rb
+- spec/cookie_store_spec.rb
+- spec/http_spec.rb
+- spec/spec_helper.rb
+- bin/anemone
 has_rdoc: true
 homepage: http://anemone.rubyforge.org
 licenses: []
@@ -74,31 +108,37 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: anemone
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Anemone web-spider framework
 test_files: 
+- spec/fakeweb_helper.rb
+- spec/page_spec.rb
 - spec/anemone_spec.rb
-- spec/cookie_store_spec.rb
 - spec/core_spec.rb
-- spec/page_spec.rb
+- spec/storage_spec.rb
 - spec/page_store_spec.rb
+- spec/cookie_store_spec.rb
 - spec/http_spec.rb
-- spec/storage_spec.rb
-- spec/fakeweb_helper.rb
 - spec/spec_helper.rb