RubyGems - sutch-anemone - Versions diffs - 0.7.2 - Mend

sutch-anemone 0.7.2

Files changed (42) hide show

checksums.yaml +15 -0
data/CHANGELOG.rdoc +136 -0
data/LICENSE.txt +19 -0
data/README.rdoc +38 -0
data/Rakefile +23 -0
data/VERSION +1 -0
data/bin/anemone +4 -0
data/lib/anemone.rb +2 -0
data/lib/anemone/cli.rb +24 -0
data/lib/anemone/cli/count.rb +22 -0
data/lib/anemone/cli/cron.rb +90 -0
data/lib/anemone/cli/pagedepth.rb +32 -0
data/lib/anemone/cli/serialize.rb +35 -0
data/lib/anemone/cli/url_list.rb +41 -0
data/lib/anemone/cookie_store.rb +35 -0
data/lib/anemone/core.rb +339 -0
data/lib/anemone/exceptions.rb +5 -0
data/lib/anemone/http.rb +187 -0
data/lib/anemone/page.rb +217 -0
data/lib/anemone/page_store.rb +161 -0
data/lib/anemone/resource.rb +42 -0
data/lib/anemone/storage.rb +44 -0
data/lib/anemone/storage/base.rb +75 -0
data/lib/anemone/storage/exceptions.rb +15 -0
data/lib/anemone/storage/kyoto_cabinet.rb +72 -0
data/lib/anemone/storage/mongodb.rb +89 -0
data/lib/anemone/storage/pstore.rb +50 -0
data/lib/anemone/storage/redis.rb +90 -0
data/lib/anemone/storage/sqlite3.rb +90 -0
data/lib/anemone/storage/tokyo_cabinet.rb +60 -0
data/lib/anemone/tentacle.rb +39 -0
data/spec/anemone_spec.rb +16 -0
data/spec/cookie_store_spec.rb +28 -0
data/spec/core_spec.rb +344 -0
data/spec/fakeweb_helper.rb +77 -0
data/spec/http_spec.rb +19 -0
data/spec/page_spec.rb +186 -0
data/spec/page_store_spec.rb +171 -0
data/spec/resource_spec.rb +91 -0
data/spec/spec_helper.rb +9 -0
data/spec/storage_spec.rb +252 -0
metadata +281 -0

data/spec/fakeweb_helper.rb ADDED Viewed

@@ -0,0 +1,77 @@
+FakeWeb.allow_net_connect = false
+module Anemone
+  SPEC_DOMAIN = "http://www.example.com/"
+  AUTH_SPEC_DOMAIN = "http://user:pass@#{URI.parse(SPEC_DOMAIN).host}/"
+  class FakePage
+    attr_accessor :links
+    attr_accessor :hrefs
+    attr_accessor :body
+    def initialize(name = '', options = {})
+      @name = name
+      @links = [options[:links]].flatten if options.has_key?(:links)
+      @hrefs = [options[:hrefs]].flatten if options.has_key?(:hrefs)
+      @redirect = options[:redirect] if options.has_key?(:redirect)
+      @auth = options[:auth] if options.has_key?(:auth)
+      @base = options[:base] if options.has_key?(:base)
+      @content_type = options[:content_type] || "text/html"
+      @body = options[:body]
+      create_body unless @body
+      add_to_fakeweb
+    end
+    def url
+      SPEC_DOMAIN + @name
+    end
+    def auth_url
+      AUTH_SPEC_DOMAIN + @name
+    end
+    private
+    def create_body
+      if @base
+        @body = "<html><head><base href=\"#{@base}\"></head><body>"
+      else
+        @body = "<html><body>"
+      end
+      @links.each{|l| @body += "<a href=\"#{SPEC_DOMAIN}#{l}\"></a>"} if @links
+      @hrefs.each{|h| @body += "<a href=\"#{h}\"></a>"} if @hrefs
+      @body += "</body></html>"
+    end
+    def add_to_fakeweb
+      options = {:body => @body, :content_type => @content_type, :status => [200, "OK"]}
+      if @redirect
+        options[:status] = [301, "Permanently Moved"]
+        # only prepend SPEC_DOMAIN if a relative url (without an http scheme) was specified
+        redirect_url = (@redirect =~ /http/) ? @redirect : SPEC_DOMAIN + @redirect
+        options[:location] = redirect_url
+        # register the page this one redirects to
+        FakeWeb.register_uri(:get, redirect_url, {:body => '',
+                                                  :content_type => @content_type,
+                                                  :status => [200, "OK"]})
+      end
+      if @auth
+        unautorized_options = {
+          :body => "Unauthorized", :status => ["401", "Unauthorized"]
+        }
+        FakeWeb.register_uri(:get, SPEC_DOMAIN + @name, unautorized_options)
+        FakeWeb.register_uri(:get, AUTH_SPEC_DOMAIN + @name, options)
+      else
+        FakeWeb.register_uri(:get, SPEC_DOMAIN + @name, options)
+      end
+    end
+  end
+end
+#default root
+Anemone::FakePage.new

data/spec/http_spec.rb ADDED Viewed

@@ -0,0 +1,19 @@
+require 'spec_helper'
+module Anemone
+  describe HTTP do
+    describe "fetch_page" do
+      before(:each) do
+        FakeWeb.clean_registry
+      end
+      it "should still return a Page if an exception occurs during the HTTP connection" do
+        HTTP.stub!(:refresh_connection).and_raise(StandardError)
+        http = Anemone::HTTP.new(:page_class => Anemone::Page)
+        http.fetch_page(SPEC_DOMAIN).should be_an_instance_of(Page)
+      end
+    end
+  end
+end

data/spec/page_spec.rb ADDED Viewed

@@ -0,0 +1,186 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+module Anemone
+  describe Page do
+    before(:each) do
+      FakeWeb.clean_registry
+      @http = Anemone::HTTP.new(:page_class => Anemone::Page)
+      @page = @http.fetch_page(FakePage.new('home', :links => '1').url)
+    end
+    it "should indicate whether it successfully fetched via HTTP" do
+      @page.should respond_to(:fetched?)
+      @page.fetched?.should == true
+      fail_page = @http.fetch_page(SPEC_DOMAIN + 'fail')
+      fail_page.fetched?.should == false
+    end
+    it "should store and expose the response body of the HTTP request" do
+      body = 'test'
+      page = @http.fetch_page(FakePage.new('body_test', {:body => body}).url)
+      page.body.should == body
+    end
+    it "should record any error that occurs during fetch_page" do
+      @page.should respond_to(:error)
+      @page.error.should be_nil
+      fail_page = @http.fetch_page(SPEC_DOMAIN + 'fail')
+      fail_page.error.should_not be_nil
+    end
+    it "should store the response headers when fetching a page" do
+      @page.headers.should_not be_nil
+      @page.headers.should have_key('content-type')
+    end
+    it "should have an OpenStruct attribute for the developer to store data in" do
+      @page.data.should_not be_nil
+      @page.data.should be_an_instance_of(OpenStruct)
+      @page.data.test = 'test'
+      @page.data.test.should == 'test'
+    end
+    it "should have a Nokogori::HTML::Document attribute for the page body" do
+      @page.doc.should_not be_nil
+      @page.doc.should be_an_instance_of(Nokogiri::HTML::Document)
+    end
+    it "should indicate whether it was fetched after an HTTP redirect" do
+      @page.should respond_to(:redirect?)
+      @page.redirect?.should == false
+      @http.fetch_pages(FakePage.new('redir', :redirect => 'home').url).first.redirect?.should == true
+    end
+    it "should have a method to tell if a URI is in the same domain as the page" do
+      @page.should respond_to(:in_domain?)
+      @page.in_domain?(URI(FakePage.new('test').url)).should == true
+      @page.in_domain?(URI('http://www.other.com/')).should == false
+    end
+    it "should include the response time for the HTTP request" do
+      @page.should respond_to(:response_time)
+    end
+    it "should have the cookies received with the page" do
+      @page.should respond_to(:cookies)
+      @page.cookies.should == []
+    end
+    describe "#to_hash" do
+      it "converts the page to a hash" do
+        hash = @page.to_hash
+        hash['url'].should == @page.url.to_s
+        hash['referer'].should == @page.referer.to_s
+        hash['links'].should == @page.links.map(&:to_s)
+      end
+      context "when redirect_to is nil" do
+        it "sets 'redirect_to' to nil in the hash" do
+          @page.redirect_to.should be_nil
+          @page.to_hash[:redirect_to].should be_nil
+        end
+      end
+      context "when redirect_to is a non-nil URI" do
+        it "sets 'redirect_to' to the URI string" do
+          new_page = Page.new(URI(SPEC_DOMAIN), {:redirect_to => URI(SPEC_DOMAIN + '1')})
+          new_page.redirect_to.to_s.should == SPEC_DOMAIN + '1'
+          new_page.to_hash['redirect_to'].should == SPEC_DOMAIN + '1'
+        end
+      end
+    end
+    describe "#from_hash" do
+      it "converts from a hash to a Page" do
+        page = @page.dup
+        page.depth = 1
+        converted = Page.from_hash(page.to_hash)
+        converted.links.should == page.links
+        converted.depth.should == page.depth
+      end
+      it 'handles a from_hash with a nil redirect_to' do
+        page_hash = @page.to_hash
+        page_hash['redirect_to'] = nil
+        lambda{Page.from_hash(page_hash)}.should_not raise_error(URI::InvalidURIError)
+        Page.from_hash(page_hash).redirect_to.should be_nil
+      end
+    end
+    describe "#redirect_to" do
+      context "when the page was a redirect" do
+        it "returns a URI of the page it redirects to" do
+          new_page = Page.new(URI(SPEC_DOMAIN), {:redirect_to => URI(SPEC_DOMAIN + '1')})
+          redirect = new_page.redirect_to
+          redirect.should be_a(URI)
+          redirect.to_s.should == SPEC_DOMAIN + '1'
+        end
+      end
+    end
+    describe "#links" do
+      it "should not convert anchors to %23" do
+        page = @http.fetch_page(FakePage.new('', :body => '<a href="#top">Top</a>').url)
+        page.links.should have(1).link
+        page.links.first.to_s.should == SPEC_DOMAIN
+      end
+    end
+    it "should detect, store and expose the base url for the page head" do
+      base = "#{SPEC_DOMAIN}path/to/base_url/"
+      page = @http.fetch_page(FakePage.new('body_test', {:base => base}).url)
+      page.base.should == URI(base)
+      @page.base.should be_nil
+    end
+    it "should have a method to convert a relative url to an absolute one" do
+      @page.should respond_to(:to_absolute)
+      # Identity
+      @page.to_absolute(@page.url).should == @page.url
+      @page.to_absolute("").should == @page.url
+      # Root-ness
+      @page.to_absolute("/").should == URI("#{SPEC_DOMAIN}")
+      # Relativeness
+      relative_path = "a/relative/path"
+      @page.to_absolute(relative_path).should == URI("#{SPEC_DOMAIN}#{relative_path}")
+      deep_page = @http.fetch_page(FakePage.new('home/deep', :links => '1').url)
+      upward_relative_path = "../a/relative/path"
+      deep_page.to_absolute(upward_relative_path).should == URI("#{SPEC_DOMAIN}#{relative_path}")
+      # The base URL case
+      base_path = "path/to/base_url/"
+      base = "#{SPEC_DOMAIN}#{base_path}"
+      page = @http.fetch_page(FakePage.new('home', {:base => base}).url)
+      # Identity
+      page.to_absolute(page.url).should == page.url
+      # It should revert to the base url
+      page.to_absolute("").should_not == page.url
+      # Root-ness
+      page.to_absolute("/").should == URI("#{SPEC_DOMAIN}")
+      # Relativeness
+      relative_path = "a/relative/path"
+      page.to_absolute(relative_path).should == URI("#{base}#{relative_path}")
+      upward_relative_path = "../a/relative/path"
+      upward_base = "#{SPEC_DOMAIN}path/to/"
+      page.to_absolute(upward_relative_path).should == URI("#{upward_base}#{relative_path}")
+    end
+  end
+end

data/spec/page_store_spec.rb ADDED Viewed

@@ -0,0 +1,171 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+%w[pstore tokyo_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+module Anemone
+  describe PageStore do
+    before(:all) do
+      FakeWeb.clean_registry
+    end
+    shared_examples_for "page storage" do
+      it "should be able to compute single-source shortest paths in-place" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1', '3'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2', :links => ['4'])
+        pages << FakePage.new('3')
+        pages << FakePage.new('4')
+        # crawl, then set depths to nil
+        page_store = Anemone.crawl(pages.first.url, @opts) do |a|
+          a.after_crawl do |ps|
+            ps.each { |url, page| page.depth = nil; ps[url] = page }
+          end
+        end.pages
+        page_store.should respond_to(:shortest_paths!)
+        page_store.shortest_paths!(pages[0].url)
+        page_store[pages[0].url].depth.should == 0
+        page_store[pages[1].url].depth.should == 1
+        page_store[pages[2].url].depth.should == 1
+        page_store[pages[3].url].depth.should == 1
+        page_store[pages[4].url].depth.should == 2
+      end
+      it "should be able to remove all redirects in-place" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+        page_store.should respond_to(:uniq!)
+        page_store.uniq!
+        page_store.has_key?(pages[1].url).should == false
+        page_store.has_key?(pages[0].url).should == true
+        page_store.has_key?(pages[2].url).should == true
+      end
+      it "should be able to find pages linking to a url" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+        page_store.should respond_to(:pages_linking_to)
+        page_store.pages_linking_to(pages[2].url).size.should == 0
+        links_to_1 = page_store.pages_linking_to(pages[1].url)
+        links_to_1.size.should == 1
+        links_to_1.first.should be_an_instance_of(Page)
+        links_to_1.first.url.to_s.should == pages[0].url
+      end
+      it "should be able to find urls linking to a url" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+        page_store.should respond_to(:pages_linking_to)
+        page_store.urls_linking_to(pages[2].url).size.should == 0
+        links_to_1 = page_store.urls_linking_to(pages[1].url)
+        links_to_1.size.should == 1
+        links_to_1.first.to_s.should == pages[0].url
+      end
+    end
+    describe Hash do
+      it_should_behave_like "page storage"
+      before(:all) do
+        @opts = {}
+      end
+    end
+    describe Storage::PStore do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @test_file = 'test.pstore'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => Storage.PStore(@test_file)}
+      end
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+    describe Storage::TokyoCabinet do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @test_file = 'test.tch'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.TokyoCabinet(@test_file)}
+      end
+      after(:each) do
+        @store.close
+      end
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+    describe Storage::SQLite3 do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @test_file = 'test.db'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+      end
+      after(:each) do
+        @store.close
+      end
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+    describe Storage::MongoDB do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @opts = {:storage => @store = Storage.MongoDB}
+      end
+      after(:each) do
+        @store.close
+      end
+    end
+    describe Storage::Redis do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @opts = {:storage => @store = Storage.Redis}
+      end
+      after(:each) do
+        @store.close
+      end
+    end
+  end
+end