anemone 0.6.1 → 0.7.0

data/CHANGELOG.rdoc

@@ -1,6 +1,23 @@
+== 0.7.0 / 2012-01-19
+
+* Major enhancements
+
+  * Added support for SQLite3 and Kyoto Cabinet storage
+
+* Minor enhancements
+
+  * Added Page#base to use base HTML element
+  * Use bundler for development dependencies
+
+* Bug fixes
+
+  * Encode characters in URLs
+  * Fix specs to run under rake
+  * Fix handling of redirect_to in storage adapters
+
 == 0.6.1 / 2011-02-24
 
-*Bug fixes
+* Bug fixes
 
   * Fix a bug preventing SSL connections from working
 

data/README.rdoc

@@ -16,7 +16,7 @@ See http://anemone.rubyforge.org for more information.
 * Records response time for each page
 * CLI program can list all pages in a domain, calculate page depths, and more
 * Obey robots.txt
-* In-memory or persistent storage of pages during crawl, using TokyoCabinet, MongoDB, or Redis
+* In-memory or persistent storage of pages during crawl, using TokyoCabinet, SQLite3, MongoDB, or Redis
 
 == Examples
 See the scripts under the <tt>lib/anemone/cli</tt> directory for examples of several useful Anemone tasks.
@@ -32,5 +32,6 @@ To test and develop this gem, additional requirements are:
 * tokyocabinet
 * mongo
 * redis
+* sqlite3
 
 You will need to have {Tokyo Cabinet}[http://fallabs.com/tokyocabinet/], {MongoDB}[http://www.mongodb.org/], and {Redis}[http://code.google.com/p/redis/] installed on your system and running.

data/Rakefile

@@ -1,26 +1,25 @@
 require 'rubygems'
 require 'rake'
+require 'rspec/core/rake_task'
 
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:rspec) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
 end
 
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
+RSpec::Core::RakeTask.new(:rcov) do |spec|
   spec.pattern = 'spec/**/*_spec.rb'
   spec.rcov = true
 end
 
-task :default => :spec
+task :default => :rspec
 
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
   version = File.exist?('VERSION') ? File.read('VERSION') : ""
 
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title = "anemone #{version}"
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
-end
+end

data/VERSION

@@ -1 +1 @@
-0.6.1
+0.7.0

data/lib/anemone/core.rb

@@ -9,7 +9,7 @@ require 'anemone/storage/base'
 
 module Anemone
 
-  VERSION = '0.6.1';
+  VERSION = '0.7.0';
 
   #
   # Convenience method to start a crawl

data/lib/anemone/http.rb

@@ -112,7 +112,7 @@ module Anemone
 
           response, response_time = get_response(loc, referer)
           code = Integer(response.code)
-          redirect_to = response.is_a?(Net::HTTPRedirection) ?  URI(response['location']).normalize : nil
+          redirect_to = response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
           yield response, code, loc, redirect_to, response_time
           limit -= 1
       end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0

data/lib/anemone/page.rb

@@ -62,7 +62,7 @@ module Anemone
       doc.search("//a[@href]").each do |a|
         u = a['href']
         next if u.nil? or u.empty?
-        abs = to_absolute(URI(u)) rescue next
+        abs = to_absolute(URI(URI.escape(u))) rescue next
         @links << abs if in_domain?(abs)
       end
       @links.uniq!
@@ -131,6 +131,21 @@ module Anemone
       404 == @code
     end
 
+    #
+    # Base URI from the HTML doc head element
+    # http://www.w3.org/TR/html4/struct/links.html#edef-BASE
+    #
+    def base
+      @base = if doc
+        href = doc.search('//head/base/@href')
+        URI(href.to_s) unless href.nil? rescue nil
+      end unless @base
+      
+      return nil if @base && @base.to_s().empty?
+      @base
+    end
+
+
     #
     # Converts relative URL *link* into an absolute URL based on the
     # location of the page
@@ -142,7 +157,7 @@ module Anemone
       link = URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/,'')))
 
       relative = URI(link)
-      absolute = @url.merge(relative)
+      absolute = base ? base.merge(relative) : @url.merge(relative)
 
       absolute.path = '/' if absolute.path.empty?
 
@@ -190,7 +205,7 @@ module Anemone
        '@visited' => hash['visited'],
        '@depth' => hash['depth'].to_i,
        '@referer' => hash['referer'],
-       '@redirect_to' => URI(hash['redirect_to']),
+       '@redirect_to' => (!!hash['redirect_to'] && !hash['redirect_to'].empty?) ? URI(hash['redirect_to']) : nil,
        '@response_time' => hash['response_time'].to_i,
        '@fetched' => hash['fetched']
       }.each do |var, value|

data/lib/anemone/storage.rb

@@ -18,6 +18,11 @@ module Anemone
       self::TokyoCabinet.new(file)
     end
 
+    def self.KyotoCabinet(file = 'anemone.tch')
+      require 'anemone/storage/kyoto_cabinet'
+      self::KyotoCabinet.new(file)
+    end
+
     def self.MongoDB(mongo_db = nil, collection_name = 'pages')
       require 'anemone/storage/mongodb'
       mongo_db ||= Mongo::Connection.new.db('anemone')
@@ -29,6 +34,11 @@ module Anemone
       require 'anemone/storage/redis'
       self::Redis.new(opts)
     end
+    
+    def self.SQLite3(file = 'anemone.db')
+      require 'anemone/storage/sqlite3'
+      self::SQLite3.new(file)
+    end
 
   end
 end

data/lib/anemone/storage/kyoto_cabinet.rb

@@ -0,0 +1,72 @@
+begin
+  require 'kyotocabinet'
+rescue LoadError
+  puts $!
+  puts "You need the kyotocabinet-ruby gem to use Anemone::Storage::KyotoCabinet"
+  exit
+end
+
+require 'forwardable'
+
+module Anemone
+  module Storage
+    class KyotoCabinet
+      extend Forwardable
+
+      def_delegators :@db, :close, :size, :each
+
+      def initialize(file)
+        raise "KyotoCabinet filename must have .kch extension" if File.extname(file) != '.kch'
+        @db = ::KyotoCabinet::DB::new
+        @db.open(file, ::KyotoCabinet::DB::OWRITER | ::KyotoCabinet::DB::OCREATE)
+        @db.clear
+      end
+
+      def [](key)
+        if value = @db[key]
+          load_value(value)
+        end
+      end
+
+      def []=(key, value)
+        @db[key] = [Marshal.dump(value)].pack("m")
+      end
+
+      def each
+        @db.each do |k, v|
+          yield(k, load_value(v))
+        end
+      end
+
+      def has_key?(key)
+        # Kyoto Cabinet doesn't have a way to query whether a key exists, so hack it
+        keys = @db.match_prefix(key)
+        !!keys && keys.include?(key)
+      end
+
+      def keys
+        acc = []
+        @db.each_key { |key| acc << key.first }
+        acc
+      end
+
+      def delete(key)
+        value = self[key]
+        @db.delete(key)
+        value
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      private
+
+      def load_value(value)
+        Marshal.load(value.unpack("m")[0])
+      end
+
+    end
+  end
+end

data/lib/anemone/storage/sqlite3.rb

@@ -0,0 +1,90 @@
+begin
+  require 'sqlite3'
+rescue LoadError
+  puts "You need the sqlite3 gem to use Anemone::Storage::SQLite3"
+  exit
+end
+
+module Anemone
+  module Storage
+    class SQLite3 
+
+      def initialize(file)
+        @db = ::SQLite3::Database.new(file)
+        create_schema
+      end
+
+      def [](url)
+        value = @db.get_first_value('SELECT data FROM anemone_storage WHERE key = ?', url.to_s)
+        if value
+          Marshal.load(value)
+        end
+      end
+
+      def []=(url, value)
+        data = Marshal.dump(value)
+        if has_key?(url)
+          @db.execute('UPDATE anemone_storage SET data = ? WHERE key = ?', data, url.to_s)
+        else
+          @db.execute('INSERT INTO anemone_storage (data, key) VALUES(?, ?)', data, url.to_s)
+        end
+      end
+
+      def delete(url)
+        page = self[url]
+        @db.execute('DELETE FROM anemone_storage WHERE key = ?', url.to_s)
+        page
+      end
+
+      def each
+        @db.execute("SELECT key, data FROM anemone_storage ORDER BY id") do |row|
+          value = Marshal.load(row[1])
+          yield row[0], value
+        end
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      def size
+        @db.get_first_value('SELECT COUNT(id) FROM anemone_storage')
+      end
+
+      def keys
+        @db.execute("SELECT key FROM anemone_storage ORDER BY id").map{|t| t[0]}
+      end
+
+      def has_key?(url)
+        !!@db.get_first_value('SELECT id FROM anemone_storage WHERE key = ?', url.to_s)
+      end
+
+      def close
+        @db.close
+      end
+
+      private
+      
+      def create_schema
+        @db.execute_batch <<SQL
+          create table if not exists anemone_storage (
+            id INTEGER PRIMARY KEY ASC,
+            key TEXT,
+            data BLOB
+          );
+          create index  if not exists anemone_key_idx on anemone_storage (key);
+SQL
+      end
+
+      def load_page(hash)
+        BINARY_FIELDS.each do |field|
+          hash[field] = hash[field].to_s
+        end
+        Page.from_hash(hash)
+      end
+
+    end
+  end
+end
+

data/lib/anemone/storage/tokyo_cabinet.rb

@@ -1,6 +1,7 @@
 begin
   require 'tokyocabinet'
 rescue LoadError
+  puts $!
   puts "You need the tokyocabinet gem to use Anemone::Storage::TokyoCabinet"
   exit
 end

data/spec/core_spec.rb

@@ -1,6 +1,6 @@
 $:.unshift(File.dirname(__FILE__))
 require 'spec_helper'
-%w[pstore tokyo_cabinet].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet sqlite3].each { |file| require "anemone/storage/#{file}.rb" }
 
 module Anemone
   describe Core do
@@ -277,6 +277,27 @@ module Anemone
       end
     end
 
+    describe Storage::SQLite3 do
+      it_should_behave_like "crawl"
+
+      before(:all) do
+        @test_file = 'test.db'
+      end
+
+      before(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+      end
+
+      after(:each) do
+        @store.close
+      end
+
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
     describe "options" do
       it "should accept options for the crawl" do
         core = Anemone.crawl(SPEC_DOMAIN, :verbose => false,

data/spec/fakeweb_helper.rb

@@ -1,10 +1,3 @@
-begin
-  require 'fakeweb'
-rescue LoadError
-  warn "You need the 'fakeweb' gem installed to test Anemone"
-  exit
-end
-
 FakeWeb.allow_net_connect = false
 
 module Anemone
@@ -22,6 +15,7 @@ module Anemone
       @hrefs = [options[:hrefs]].flatten if options.has_key?(:hrefs)
       @redirect = options[:redirect] if options.has_key?(:redirect)
       @auth = options[:auth] if options.has_key?(:auth)
+      @base = options[:base] if options.has_key?(:base)      
       @content_type = options[:content_type] || "text/html"
       @body = options[:body]
 
@@ -40,7 +34,11 @@ module Anemone
     private
 
     def create_body
-      @body = "<html><body>"
+      if @base
+        @body = "<html><head><base href=\"#{@base}\"></head><body>"
+      else
+        @body = "<html><body>"
+      end
       @links.each{|l| @body += "<a href=\"#{SPEC_DOMAIN}#{l}\"></a>"} if @links
       @hrefs.each{|h| @body += "<a href=\"#{h}\"></a>"} if @hrefs
       @body += "</body></html>"

data/spec/http_spec.rb

@@ -1,4 +1,3 @@
-$:.unshift(File.dirname(__FILE__))
 require 'spec_helper'
 
 module Anemone
@@ -10,12 +9,7 @@ module Anemone
       end
 
       it "should still return a Page if an exception occurs during the HTTP connection" do
-        class HTTP
-          def refresh_connection
-            raise "test exception"
-          end
-        end
-
+        HTTP.stub!(:refresh_connection).and_raise(StandardError)
         http = Anemone::HTTP.new
         http.fetch_page(SPEC_DOMAIN).should be_an_instance_of(Page)
       end

data/spec/page_spec.rb

@@ -74,19 +74,103 @@ module Anemone
       @page.cookies.should == []
     end
 
-    it "should have a to_hash method that converts the page to a hash" do
-      hash = @page.to_hash
-      hash['url'].should == @page.url.to_s
-      hash['referer'].should == @page.referer.to_s
-      hash['links'].should == @page.links.map(&:to_s)
-    end
-
-    it "should have a from_hash method to convert from a hash to a Page" do
-      page = @page.dup
-      page.depth = 1
-      converted = Page.from_hash(page.to_hash)
-      converted.links.should == page.links
-      converted.depth.should == page.depth
+    describe "#to_hash" do
+      it "converts the page to a hash" do
+        hash = @page.to_hash
+        hash['url'].should == @page.url.to_s
+        hash['referer'].should == @page.referer.to_s
+        hash['links'].should == @page.links.map(&:to_s)
+      end
+
+      context "when redirect_to is nil" do
+        it "sets 'redirect_to' to nil in the hash" do
+          @page.redirect_to.should be_nil
+          @page.to_hash[:redirect_to].should be_nil
+        end
+      end
+
+      context "when redirect_to is a non-nil URI" do
+        it "sets 'redirect_to' to the URI string" do
+          new_page = Page.new(URI(SPEC_DOMAIN), {:redirect_to => URI(SPEC_DOMAIN + '1')})
+          new_page.redirect_to.to_s.should == SPEC_DOMAIN + '1'
+          new_page.to_hash['redirect_to'].should == SPEC_DOMAIN + '1'
+        end
+      end
+    end
+
+    describe "#from_hash" do
+      it "converts from a hash to a Page" do
+        page = @page.dup
+        page.depth = 1
+        converted = Page.from_hash(page.to_hash)
+        converted.links.should == page.links
+        converted.depth.should == page.depth
+      end
+
+      it 'handles a from_hash with a nil redirect_to' do
+        page_hash = @page.to_hash
+        page_hash['redirect_to'] = nil
+        lambda{Page.from_hash(page_hash)}.should_not raise_error(URI::InvalidURIError)
+        Page.from_hash(page_hash).redirect_to.should be_nil
+      end
+    end
+
+    describe "#redirect_to" do
+      context "when the page was a redirect" do
+        it "returns a URI of the page it redirects to" do
+          new_page = Page.new(URI(SPEC_DOMAIN), {:redirect_to => URI(SPEC_DOMAIN + '1')})
+          redirect = new_page.redirect_to
+          redirect.should be_a(URI)
+          redirect.to_s.should == SPEC_DOMAIN + '1'
+        end
+      end
+    end
+
+    it "should detect, store and expose the base url for the page head" do
+      base = "#{SPEC_DOMAIN}path/to/base_url/"
+      page = @http.fetch_page(FakePage.new('body_test', {:base => base}).url)
+      page.base.should == URI(base)
+      @page.base.should be_nil
+    end
+
+    it "should have a method to convert a relative url to an absolute one" do
+      @page.should respond_to(:to_absolute)
+      
+      # Identity
+      @page.to_absolute(@page.url).should == @page.url
+      @page.to_absolute("").should == @page.url
+      
+      # Root-ness
+      @page.to_absolute("/").should == URI("#{SPEC_DOMAIN}")
+      
+      # Relativeness
+      relative_path = "a/relative/path"
+      @page.to_absolute(relative_path).should == URI("#{SPEC_DOMAIN}#{relative_path}")
+      
+      deep_page = @http.fetch_page(FakePage.new('home/deep', :links => '1').url)
+      upward_relative_path = "../a/relative/path"
+      deep_page.to_absolute(upward_relative_path).should == URI("#{SPEC_DOMAIN}#{relative_path}")
+      
+      # The base URL case
+      base_path = "path/to/base_url/"
+      base = "#{SPEC_DOMAIN}#{base_path}"
+      page = @http.fetch_page(FakePage.new('home', {:base => base}).url)
+      
+      # Identity
+      page.to_absolute(page.url).should == page.url
+      # It should revert to the base url
+      page.to_absolute("").should_not == page.url
+
+      # Root-ness
+      page.to_absolute("/").should == URI("#{SPEC_DOMAIN}")
+      
+      # Relativeness
+      relative_path = "a/relative/path"
+      page.to_absolute(relative_path).should == URI("#{base}#{relative_path}")
+      
+      upward_relative_path = "../a/relative/path"
+      upward_base = "#{SPEC_DOMAIN}path/to/"
+      page.to_absolute(upward_relative_path).should == URI("#{upward_base}#{relative_path}")      
     end
 
   end

data/spec/page_store_spec.rb

@@ -1,6 +1,6 @@
 $:.unshift(File.dirname(__FILE__))
 require 'spec_helper'
-%w[pstore tokyo_cabinet mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
 
 module Anemone
   describe PageStore do
@@ -125,6 +125,24 @@ module Anemone
       end
     end
 
+    describe Storage::SQLite3 do
+      it_should_behave_like "page storage"
+
+      before(:each) do
+        @test_file = 'test.db'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+      end
+
+      after(:each) do
+        @store.close
+      end
+
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
     describe Storage::MongoDB do
       it_should_behave_like "page storage"
 

data/spec/spec_helper.rb

@@ -1,4 +1,6 @@
 require 'rubygems'
+require 'bundler/setup'
+require 'fakeweb'
 require File.dirname(__FILE__) + '/fakeweb_helper'
 
 $:.unshift(File.dirname(__FILE__) + '/../lib/')

data/spec/storage_spec.rb

@@ -1,7 +1,7 @@
 $:.unshift(File.dirname(__FILE__))
 require 'spec_helper'
 
-%w[pstore tokyo_cabinet mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet kyoto_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
 
 module Anemone
   describe Storage do
@@ -25,6 +25,14 @@ module Anemone
       store.close
     end
 
+    it "should have a class method to produce a SQLite3" do
+      test_file = 'test.db'
+      Anemone::Storage.should respond_to(:SQLite3)
+      store = Anemone::Storage.SQLite3(test_file)
+      store.should be_an_instance_of(Anemone::Storage::SQLite3)
+      store.close
+    end
+
     it "should have a class method to produce a MongoDB" do
       Anemone::Storage.should respond_to(:MongoDB)
       store = Anemone::Storage.MongoDB
@@ -105,6 +113,12 @@ module Anemone
 
           merged.should === @store
         end
+
+        it "should correctly deserialize nil redirect_to when loading" do
+          @page.redirect_to.should be_nil
+          @store[@url] = @page 
+          @store[@url].redirect_to.should be_nil
+        end
       end
 
       describe PStore do
@@ -143,6 +157,47 @@ module Anemone
         end
       end
 
+      describe KyotoCabinet do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.kch'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.KyotoCabinet(@test_file)
+        end
+
+        after(:each) do
+          @store.close
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+
+        it "should raise an error if supplied with a file extension other than .kch" do
+          lambda { Anemone::Storage.KyotoCabinet('test.tmp') }.should raise_error(RuntimeError)
+        end
+      end
+
+      describe SQLite3 do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.db'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.SQLite3(@test_file)
+        end
+
+        after(:each) do
+          @store.close
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+
+      end
+
       describe Storage::MongoDB do
         it_should_behave_like "storage engine"
 

metadata

@@ -1,139 +1,218 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: anemone
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 6
-  - 1
-  version: 0.6.1
+version: !ruby/object:Gem::Version
+  version: 0.7.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Chris Kite
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-02-24 00:00:00 -06:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-01-20 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: nokogiri
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &19111780 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 3
-        - 0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.3.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: robots
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *19111780
+- !ruby/object:Gem::Dependency
+  name: robots
+  requirement: &19111300 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 7
-        - 2
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.7.2
   type: :runtime
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: *19111300
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &19141340 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :development
+  prerelease: false
+  version_requirements: *19141340
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &19140880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *19140880
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: &19140420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *19140420
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &19139960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :development
+  prerelease: false
+  version_requirements: *19139960
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: &19139500 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: *19139500
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: &19139040 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: *19139040
+- !ruby/object:Gem::Dependency
+  name: tokyocabinet
+  requirement: &19138580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.29'
+  type: :development
+  prerelease: false
+  version_requirements: *19138580
+- !ruby/object:Gem::Dependency
+  name: kyotocabinet-ruby
+  requirement: &19138120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.27.1
+  type: :development
+  prerelease: false
+  version_requirements: *19138120
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: &19137660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+  type: :development
+  prerelease: false
+  version_requirements: *19137660
 description: 
 email: 
-executables: 
+executables:
 - anemone
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.rdoc
-files: 
+files:
 - VERSION
 - LICENSE.txt
 - CHANGELOG.rdoc
 - README.rdoc
 - Rakefile
-- lib/anemone.rb
-- lib/anemone/cookie_store.rb
-- lib/anemone/storage.rb
-- lib/anemone/core.rb
-- lib/anemone/cli.rb
-- lib/anemone/exceptions.rb
-- lib/anemone/tentacle.rb
-- lib/anemone/storage/tokyo_cabinet.rb
-- lib/anemone/storage/base.rb
-- lib/anemone/storage/exceptions.rb
 - lib/anemone/storage/pstore.rb
 - lib/anemone/storage/mongodb.rb
+- lib/anemone/storage/tokyo_cabinet.rb
+- lib/anemone/storage/exceptions.rb
 - lib/anemone/storage/redis.rb
-- lib/anemone/http.rb
+- lib/anemone/storage/sqlite3.rb
+- lib/anemone/storage/base.rb
+- lib/anemone/storage/kyoto_cabinet.rb
 - lib/anemone/page_store.rb
-- lib/anemone/cli/cron.rb
-- lib/anemone/cli/pagedepth.rb
-- lib/anemone/cli/count.rb
+- lib/anemone/storage.rb
+- lib/anemone/tentacle.rb
+- lib/anemone/http.rb
+- lib/anemone/cli.rb
+- lib/anemone/page.rb
+- lib/anemone/exceptions.rb
+- lib/anemone/core.rb
 - lib/anemone/cli/url_list.rb
 - lib/anemone/cli/serialize.rb
-- lib/anemone/page.rb
-- spec/http_spec.rb
-- spec/page_store_spec.rb
-- spec/core_spec.rb
+- lib/anemone/cli/count.rb
+- lib/anemone/cli/cron.rb
+- lib/anemone/cli/pagedepth.rb
+- lib/anemone/cookie_store.rb
+- lib/anemone.rb
 - spec/fakeweb_helper.rb
 - spec/page_spec.rb
-- spec/cookie_store_spec.rb
 - spec/anemone_spec.rb
-- spec/spec_helper.rb
+- spec/core_spec.rb
 - spec/storage_spec.rb
+- spec/page_store_spec.rb
+- spec/cookie_store_spec.rb
+- spec/http_spec.rb
+- spec/spec_helper.rb
 - bin/anemone
-has_rdoc: true
 homepage: http://anemone.rubyforge.org
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - -m
 - README.rdoc
 - -t
 - Anemone
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: anemone
-rubygems_version: 1.3.7
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: Anemone web-spider framework
-test_files: 
-- spec/http_spec.rb
-- spec/page_store_spec.rb
-- spec/core_spec.rb
+test_files:
 - spec/fakeweb_helper.rb
 - spec/page_spec.rb
-- spec/cookie_store_spec.rb
 - spec/anemone_spec.rb
-- spec/spec_helper.rb
+- spec/core_spec.rb
 - spec/storage_spec.rb
+- spec/page_store_spec.rb
+- spec/cookie_store_spec.rb
+- spec/http_spec.rb
+- spec/spec_helper.rb