spk-anemone 0.2.4 → 0.3.0

data/spec/http_spec.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+module Anemone
+  describe HTTP do
+
+    describe "fetch_page" do
+      before(:each) do
+        FakeWeb.clean_registry
+      end
+
+      it "should still return a Page if an exception occurs during the HTTP connection" do
+        class HTTP
+          def refresh_connection
+            raise "test exception"
+          end
+        end
+
+        http = HTTP.new
+        http.fetch_page(SPEC_DOMAIN).should be_an_instance_of(Page)
+      end
+    end
+  end
+end

data/spec/page_spec.rb

@@ -3,43 +3,57 @@ require File.dirname(__FILE__) + '/spec_helper'
 module Anemone
   describe Page do
 
-    before(:all) do
+    before(:each) do
+      FakeWeb.clean_registry
       @http = Anemone::HTTP.new
+      @page = @http.fetch_page(FakePage.new('home').url)
     end
 
-    before(:each) do
-      @page = @http.fetch_page(FakePage.new('home').url)
+    it "should indicate whether it successfully fetched via HTTP" do
+      @page.should respond_to(:fetched?)
+      @page.fetched?.should == true
+
+      fail_page = @http.fetch_page(SPEC_DOMAIN + 'fail')
+      fail_page.fetched?.should == false
     end
-    
+
+    it "should record any error that occurs during fetch_page" do
+      @page.should respond_to(:error)
+      @page.error.should be_nil
+
+      fail_page = @http.fetch_page(SPEC_DOMAIN + 'fail')
+      fail_page.error.should_not be_nil
+    end
+
     it "should store the response headers when fetching a page" do
       @page.headers.should_not be_nil
       @page.headers.should have_key('content-type')
     end
-    
+
     it "should have an OpenStruct attribute for the developer to store data in" do
       @page.data.should_not be_nil
       @page.data.should be_an_instance_of(OpenStruct)
-      
+
       @page.data.test = 'test'
       @page.data.test.should == 'test'
     end
-    
+
     it "should have a Nokogori::HTML::Document attribute for the page body" do
       @page.doc.should_not be_nil
       @page.doc.should be_an_instance_of(Nokogiri::HTML::Document)
     end
-    
+
     it "should indicate whether it was fetched after an HTTP redirect" do
       @page.should respond_to(:redirect?)
-      
+
       @page.redirect?.should == false
-      
-      @http.fetch_page(FakePage.new('redir', :redirect => 'home').url).redirect?.should == true
+
+      @http.fetch_pages(FakePage.new('redir', :redirect => 'home').url).first.redirect?.should == true
     end
-    
+
     it "should have a method to tell if a URI is in the same domain as the page" do
       @page.should respond_to(:in_domain?)
-      
+
       @page.in_domain?(URI(FakePage.new('test').url)).should == true
       @page.in_domain?(URI('http://www.other.com/')).should == false
     end
@@ -47,6 +61,6 @@ module Anemone
     it "should include the response time for the HTTP request" do
       @page.should respond_to(:response_time)
     end
-    
+
   end
 end

data/spec/page_store_spec.rb

@@ -0,0 +1,128 @@
+require File.dirname(__FILE__) + '/spec_helper'
+%w[pstore tokyo_cabinet].each { |file| require "anemone/storage/#{file}.rb" }
+
+module Anemone
+  describe PageStore do
+
+    before(:all) do
+      FakeWeb.clean_registry
+    end
+
+    shared_examples_for "page storage" do
+      it "should be able to computer single-source shortest paths in-place" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1', '3'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2', :links => ['4'])
+        pages << FakePage.new('3')
+        pages << FakePage.new('4')
+
+        # crawl, then set depths to nil
+        page_store = Anemone.crawl(pages.first.url, @opts) do |a|
+          a.after_crawl do |ps|
+            ps.each { |url, page| page.depth = nil; ps[url] = page }
+          end
+        end.pages
+
+        page_store.should respond_to(:shortest_paths!)
+
+        page_store.shortest_paths!(pages[0].url)
+        page_store[pages[0].url].depth.should == 0
+        page_store[pages[1].url].depth.should == 1
+        page_store[pages[2].url].depth.should == 1
+        page_store[pages[3].url].depth.should == 1
+        page_store[pages[4].url].depth.should == 2
+      end
+
+      it "should be able to remove all redirects in-place" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+
+        page_store.should respond_to(:uniq!)
+
+        page_store.uniq!
+        page_store.has_key?(pages[1].url).should == false
+        page_store.has_key?(pages[0].url).should == true
+        page_store.has_key?(pages[2].url).should == true
+      end
+
+      it "should be able to find pages linking to a url" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+
+        page_store.should respond_to(:pages_linking_to)
+
+        page_store.pages_linking_to(pages[2].url).size.should == 0
+        links_to_1 = page_store.pages_linking_to(pages[1].url)
+        links_to_1.size.should == 1
+        links_to_1.first.should be_an_instance_of(Page)
+        links_to_1.first.url.to_s.should == pages[0].url
+      end
+
+      it "should be able to find urls linking to a url" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+
+        page_store.should respond_to(:pages_linking_to)
+
+        page_store.urls_linking_to(pages[2].url).size.should == 0
+        links_to_1 = page_store.urls_linking_to(pages[1].url)
+        links_to_1.size.should == 1
+        links_to_1.first.to_s.should == pages[0].url
+      end
+    end
+
+    describe Hash do
+      it_should_behave_like "page storage"
+
+      before(:all) do
+        @opts = {}
+      end
+    end
+
+    describe Storage::PStore do
+      it_should_behave_like "page storage"
+
+      before(:each) do
+        @test_file = 'test.pstore'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => Storage.PStore(@test_file)}
+      end
+
+      after(:all) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
+    describe Storage::TokyoCabinet do
+      it_should_behave_like "page storage"
+
+      before(:each) do
+        @test_file = 'test.tch'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.TokyoCabinet(@test_file)}
+      end
+
+      after(:each) do
+        @store.close
+      end
+
+      after(:all) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
+  end
+end

data/spec/storage_spec.rb

@@ -0,0 +1,123 @@
+require File.dirname(__FILE__) + '/spec_helper'
+%w[pstore tokyo_cabinet].each { |file| require "anemone/storage/#{file}.rb" }
+
+module Anemone
+  describe Storage do
+
+    it "should have a class method to produce a Hash" do
+      Anemone::Storage.should respond_to(:Hash)
+      Anemone::Storage.Hash.should be_an_instance_of(Hash)
+    end
+
+    it "should have a class method to produce a PStore" do
+      test_file = 'test.pstore'
+      Anemone::Storage.should respond_to(:PStore)
+      Anemone::Storage.PStore(test_file).should be_an_instance_of(Anemone::Storage::PStore)
+    end
+
+    it "should have a class method to produce a TokyoCabinet" do
+      test_file = 'test.tch'
+      Anemone::Storage.should respond_to(:TokyoCabinet)
+      store = Anemone::Storage.TokyoCabinet(test_file)
+      store.should be_an_instance_of(Anemone::Storage::TokyoCabinet)
+      store.close
+    end
+
+    module Storage
+      shared_examples_for "storage engine" do
+        it "should implement [] and []=" do
+          @store.should respond_to(:[])
+          @store.should respond_to(:[]=)
+
+          @store['index'] = 'test'
+          @store['index'].should == 'test'
+        end
+
+        it "should implement has_key?" do
+          @store.should respond_to(:has_key?)
+
+          @store['index'] = 'test'
+          @store.has_key?('index').should == true
+
+          @store.has_key?('missing').should == false
+        end
+
+        it "should implement delete" do
+          @store.should respond_to(:delete)
+
+          @store['index'] = 'test'
+          @store.delete('index').should == 'test'
+          @store.has_key?('index').should  == false
+        end
+
+        it "should implement keys" do
+          @store.should respond_to(:keys)
+
+          keys = ['a', 'b', 'c']
+          keys.each { |key| @store[key] = key }
+
+          @store.keys.should == keys
+        end
+
+        it "should implement each" do
+          @store.should respond_to(:each)
+
+          keys = ['a', 'b', 'c']
+          keys.each { |key| @store[key] = key }
+
+          result = {}
+          @store.each { |k, v| result[k] = v }
+          result.values.should == keys
+        end
+
+        it "should implement merge!, and return self" do
+          @store.should respond_to(:merge!)
+
+          hash = {'a' => 'a', 'b' => 'b', 'c' => 'c'}
+          merged = @store.merge! hash
+          hash.each { |key, value| @store[key].should == value }
+
+          merged.should === @store
+        end
+      end
+
+      describe PStore do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.pstore'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.PStore(@test_file)
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+      end
+
+      describe TokyoCabinet do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.tch'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.TokyoCabinet(@test_file)
+        end
+
+        after(:each) do
+          @store.close
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+
+        it "should raise an error if supplied with a file extension other than .tch" do
+          lambda { Anemone::Storage.TokyoCabinet('test.tmp') }.should raise_error(RuntimeError)
+        end
+
+      end
+
+    end
+  end
+end

metadata

@@ -1,16 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: spk-anemone
 version: !ruby/object:Gem::Version 
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors: 
 - Chris Kite
-- spk
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2009-11-26 00:00:00 +01:00
+date: 2010-01-03 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -21,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.4.0
+        version: 1.4.1
     version: 
 - !ruby/object:Gem::Dependency 
   name: robots
@@ -50,8 +49,11 @@ files:
 - lib/anemone/core.rb
 - lib/anemone/http.rb
 - lib/anemone/page.rb
-- lib/anemone/page_hash.rb
+- lib/anemone/page_store.rb
 - lib/anemone/tentacle.rb
+- lib/anemone/storage.rb
+- lib/anemone/storage/pstore.rb
+- lib/anemone/storage/tokyo_cabinet.rb
 - lib/anemone/cli.rb
 - lib/anemone/cli/url_list.rb
 - lib/anemone/cli/cron.rb
@@ -93,5 +95,8 @@ test_files:
 - spec/anemone_spec.rb
 - spec/core_spec.rb
 - spec/page_spec.rb
+- spec/page_store_spec.rb
+- spec/http_spec.rb
+- spec/storage_spec.rb
 - spec/fakeweb_helper.rb
 - spec/spec_helper.rb