RubyGems - sutch-anemone - Versions diffs - 0.7.2 - Mend

sutch-anemone 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (42) hide show

checksums.yaml +15 -0
data/CHANGELOG.rdoc +136 -0
data/LICENSE.txt +19 -0
data/README.rdoc +38 -0
data/Rakefile +23 -0
data/VERSION +1 -0
data/bin/anemone +4 -0
data/lib/anemone.rb +2 -0
data/lib/anemone/cli.rb +24 -0
data/lib/anemone/cli/count.rb +22 -0
data/lib/anemone/cli/cron.rb +90 -0
data/lib/anemone/cli/pagedepth.rb +32 -0
data/lib/anemone/cli/serialize.rb +35 -0
data/lib/anemone/cli/url_list.rb +41 -0
data/lib/anemone/cookie_store.rb +35 -0
data/lib/anemone/core.rb +339 -0
data/lib/anemone/exceptions.rb +5 -0
data/lib/anemone/http.rb +187 -0
data/lib/anemone/page.rb +217 -0
data/lib/anemone/page_store.rb +161 -0
data/lib/anemone/resource.rb +42 -0
data/lib/anemone/storage.rb +44 -0
data/lib/anemone/storage/base.rb +75 -0
data/lib/anemone/storage/exceptions.rb +15 -0
data/lib/anemone/storage/kyoto_cabinet.rb +72 -0
data/lib/anemone/storage/mongodb.rb +89 -0
data/lib/anemone/storage/pstore.rb +50 -0
data/lib/anemone/storage/redis.rb +90 -0
data/lib/anemone/storage/sqlite3.rb +90 -0
data/lib/anemone/storage/tokyo_cabinet.rb +60 -0
data/lib/anemone/tentacle.rb +39 -0
data/spec/anemone_spec.rb +16 -0
data/spec/cookie_store_spec.rb +28 -0
data/spec/core_spec.rb +344 -0
data/spec/fakeweb_helper.rb +77 -0
data/spec/http_spec.rb +19 -0
data/spec/page_spec.rb +186 -0
data/spec/page_store_spec.rb +171 -0
data/spec/resource_spec.rb +91 -0
data/spec/spec_helper.rb +9 -0
data/spec/storage_spec.rb +252 -0
metadata +281 -0

data/spec/fakeweb_helper.rb ADDED Viewed

@@ -0,0 +1,77 @@
+FakeWeb.allow_net_connect = false
+module Anemone
+  SPEC_DOMAIN = "http://www.example.com/"
+  AUTH_SPEC_DOMAIN = "http://user:pass@#{URI.parse(SPEC_DOMAIN).host}/"
+  class FakePage
+    attr_accessor :links
+    attr_accessor :hrefs
+    attr_accessor :body
+    def initialize(name = '', options = {})
+      @name = name
+      @links = [options[:links]].flatten if options.has_key?(:links)
+      @hrefs = [options[:hrefs]].flatten if options.has_key?(:hrefs)
+      @redirect = options[:redirect] if options.has_key?(:redirect)
+      @auth = options[:auth] if options.has_key?(:auth)
+      @base = options[:base] if options.has_key?(:base)
+      @content_type = options[:content_type] || "text/html"
+      @body = options[:body]
+      create_body unless @body
+      add_to_fakeweb
+    end
+    def url
+      SPEC_DOMAIN + @name
+    end
+    def auth_url
+      AUTH_SPEC_DOMAIN + @name
+    end
+    private
+    def create_body
+      if @base
+        @body = "<html><head><base href=\"#{@base}\"></head><body>"
+      else
+        @body = "<html><body>"
+      end
+      @links.each{|l| @body += "<a href=\"#{SPEC_DOMAIN}#{l}\"></a>"} if @links
+      @hrefs.each{|h| @body += "<a href=\"#{h}\"></a>"} if @hrefs
+      @body += "</body></html>"
+    end
+    def add_to_fakeweb
+      options = {:body => @body, :content_type => @content_type, :status => [200, "OK"]}
+      if @redirect
+        options[:status] = [301, "Permanently Moved"]
+        # only prepend SPEC_DOMAIN if a relative url (without an http scheme) was specified
+        redirect_url = (@redirect =~ /http/) ? @redirect : SPEC_DOMAIN + @redirect
+        options[:location] = redirect_url
+        # register the page this one redirects to
+        FakeWeb.register_uri(:get, redirect_url, {:body => '',
+                                                  :content_type => @content_type,
+                                                  :status => [200, "OK"]})
+      end
+      if @auth
+        unautorized_options = {
+          :body => "Unauthorized", :status => ["401", "Unauthorized"]
+        }
+        FakeWeb.register_uri(:get, SPEC_DOMAIN + @name, unautorized_options)
+        FakeWeb.register_uri(:get, AUTH_SPEC_DOMAIN + @name, options)
+      else
+        FakeWeb.register_uri(:get, SPEC_DOMAIN + @name, options)
+      end
+    end
+  end
+end
+#default root
+Anemone::FakePage.new

data/spec/http_spec.rb ADDED Viewed

@@ -0,0 +1,19 @@
+require 'spec_helper'
+module Anemone
+  describe HTTP do
+    describe "fetch_page" do
+      before(:each) do
+        FakeWeb.clean_registry
+      end
+      it "should still return a Page if an exception occurs during the HTTP connection" do
+        HTTP.stub!(:refresh_connection).and_raise(StandardError)
+        http = Anemone::HTTP.new(:page_class => Anemone::Page)
+        http.fetch_page(SPEC_DOMAIN).should be_an_instance_of(Page)
+      end
+    end
+  end
+end

data/spec/page_spec.rb ADDED Viewed

@@ -0,0 +1,186 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+module Anemone
+  describe Page do
+    before(:each) do
+      FakeWeb.clean_registry
+      @http = Anemone::HTTP.new(:page_class => Anemone::Page)
+      @page = @http.fetch_page(FakePage.new('home', :links => '1').url)
+    end
+    it "should indicate whether it successfully fetched via HTTP" do
+      @page.should respond_to(:fetched?)
+      @page.fetched?.should == true
+      fail_page = @http.fetch_page(SPEC_DOMAIN + 'fail')
+      fail_page.fetched?.should == false
+    end
+    it "should store and expose the response body of the HTTP request" do
+      body = 'test'
+      page = @http.fetch_page(FakePage.new('body_test', {:body => body}).url)
+      page.body.should == body
+    end
+    it "should record any error that occurs during fetch_page" do
+      @page.should respond_to(:error)
+      @page.error.should be_nil
+      fail_page = @http.fetch_page(SPEC_DOMAIN + 'fail')
+      fail_page.error.should_not be_nil
+    end
+    it "should store the response headers when fetching a page" do
+      @page.headers.should_not be_nil
+      @page.headers.should have_key('content-type')
+    end
+    it "should have an OpenStruct attribute for the developer to store data in" do
+      @page.data.should_not be_nil
+      @page.data.should be_an_instance_of(OpenStruct)
+      @page.data.test = 'test'
+      @page.data.test.should == 'test'
+    end
+    it "should have a Nokogori::HTML::Document attribute for the page body" do
+      @page.doc.should_not be_nil
+      @page.doc.should be_an_instance_of(Nokogiri::HTML::Document)
+    end
+    it "should indicate whether it was fetched after an HTTP redirect" do
+      @page.should respond_to(:redirect?)
+      @page.redirect?.should == false
+      @http.fetch_pages(FakePage.new('redir', :redirect => 'home').url).first.redirect?.should == true
+    end
+    it "should have a method to tell if a URI is in the same domain as the page" do
+      @page.should respond_to(:in_domain?)
+      @page.in_domain?(URI(FakePage.new('test').url)).should == true
+      @page.in_domain?(URI('http://www.other.com/')).should == false
+    end
+    it "should include the response time for the HTTP request" do
+      @page.should respond_to(:response_time)
+    end
+    it "should have the cookies received with the page" do
+      @page.should respond_to(:cookies)
+      @page.cookies.should == []
+    end
+    describe "#to_hash" do
+      it "converts the page to a hash" do
+        hash = @page.to_hash
+        hash['url'].should == @page.url.to_s
+        hash['referer'].should == @page.referer.to_s
+        hash['links'].should == @page.links.map(&:to_s)
+      end
+      context "when redirect_to is nil" do
+        it "sets 'redirect_to' to nil in the hash" do
+          @page.redirect_to.should be_nil
+          @page.to_hash[:redirect_to].should be_nil
+        end
+      end
+      context "when redirect_to is a non-nil URI" do
+        it "sets 'redirect_to' to the URI string" do
+          new_page = Page.new(URI(SPEC_DOMAIN), {:redirect_to => URI(SPEC_DOMAIN + '1')})
+          new_page.redirect_to.to_s.should == SPEC_DOMAIN + '1'
+          new_page.to_hash['redirect_to'].should == SPEC_DOMAIN + '1'
+        end
+      end
+    end
+    describe "#from_hash" do
+      it "converts from a hash to a Page" do
+        page = @page.dup
+        page.depth = 1
+        converted = Page.from_hash(page.to_hash)
+        converted.links.should == page.links
+        converted.depth.should == page.depth
+      end
+      it 'handles a from_hash with a nil redirect_to' do
+        page_hash = @page.to_hash
+        page_hash['redirect_to'] = nil
+        lambda{Page.from_hash(page_hash)}.should_not raise_error(URI::InvalidURIError)
+        Page.from_hash(page_hash).redirect_to.should be_nil
+      end
+    end
+    describe "#redirect_to" do
+      context "when the page was a redirect" do
+        it "returns a URI of the page it redirects to" do
+          new_page = Page.new(URI(SPEC_DOMAIN), {:redirect_to => URI(SPEC_DOMAIN + '1')})
+          redirect = new_page.redirect_to
+          redirect.should be_a(URI)
+          redirect.to_s.should == SPEC_DOMAIN + '1'
+        end
+      end
+    end
+    describe "#links" do
+      it "should not convert anchors to %23" do
+        page = @http.fetch_page(FakePage.new('', :body => '<a href="#top">Top</a>').url)
+        page.links.should have(1).link
+        page.links.first.to_s.should == SPEC_DOMAIN
+      end
+    end
+    it "should detect, store and expose the base url for the page head" do
+      base = "#{SPEC_DOMAIN}path/to/base_url/"
+      page = @http.fetch_page(FakePage.new('body_test', {:base => base}).url)
+      page.base.should == URI(base)
+      @page.base.should be_nil
+    end
+    it "should have a method to convert a relative url to an absolute one" do
+      @page.should respond_to(:to_absolute)
+      # Identity
+      @page.to_absolute(@page.url).should == @page.url
+      @page.to_absolute("").should == @page.url
+      # Root-ness
+      @page.to_absolute("/").should == URI("#{SPEC_DOMAIN}")
+      # Relativeness
+      relative_path = "a/relative/path"
+      @page.to_absolute(relative_path).should == URI("#{SPEC_DOMAIN}#{relative_path}")
+      deep_page = @http.fetch_page(FakePage.new('home/deep', :links => '1').url)
+      upward_relative_path = "../a/relative/path"
+      deep_page.to_absolute(upward_relative_path).should == URI("#{SPEC_DOMAIN}#{relative_path}")
+      # The base URL case
+      base_path = "path/to/base_url/"
+      base = "#{SPEC_DOMAIN}#{base_path}"
+      page = @http.fetch_page(FakePage.new('home', {:base => base}).url)
+      # Identity
+      page.to_absolute(page.url).should == page.url
+      # It should revert to the base url
+      page.to_absolute("").should_not == page.url
+      # Root-ness
+      page.to_absolute("/").should == URI("#{SPEC_DOMAIN}")
+      # Relativeness
+      relative_path = "a/relative/path"
+      page.to_absolute(relative_path).should == URI("#{base}#{relative_path}")
+      upward_relative_path = "../a/relative/path"
+      upward_base = "#{SPEC_DOMAIN}path/to/"
+      page.to_absolute(upward_relative_path).should == URI("#{upward_base}#{relative_path}")
+    end
+  end
+end

data/spec/page_store_spec.rb ADDED Viewed

@@ -0,0 +1,171 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+%w[pstore tokyo_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+module Anemone
+  describe PageStore do
+    before(:all) do
+      FakeWeb.clean_registry
+    end
+    shared_examples_for "page storage" do
+      it "should be able to compute single-source shortest paths in-place" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1', '3'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2', :links => ['4'])
+        pages << FakePage.new('3')
+        pages << FakePage.new('4')
+        # crawl, then set depths to nil
+        page_store = Anemone.crawl(pages.first.url, @opts) do |a|
+          a.after_crawl do |ps|
+            ps.each { |url, page| page.depth = nil; ps[url] = page }
+          end
+        end.pages
+        page_store.should respond_to(:shortest_paths!)
+        page_store.shortest_paths!(pages[0].url)
+        page_store[pages[0].url].depth.should == 0
+        page_store[pages[1].url].depth.should == 1
+        page_store[pages[2].url].depth.should == 1
+        page_store[pages[3].url].depth.should == 1
+        page_store[pages[4].url].depth.should == 2
+      end
+      it "should be able to remove all redirects in-place" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+        page_store.should respond_to(:uniq!)
+        page_store.uniq!
+        page_store.has_key?(pages[1].url).should == false
+        page_store.has_key?(pages[0].url).should == true
+        page_store.has_key?(pages[2].url).should == true
+      end
+      it "should be able to find pages linking to a url" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+        page_store.should respond_to(:pages_linking_to)
+        page_store.pages_linking_to(pages[2].url).size.should == 0
+        links_to_1 = page_store.pages_linking_to(pages[1].url)
+        links_to_1.size.should == 1
+        links_to_1.first.should be_an_instance_of(Page)
+        links_to_1.first.url.to_s.should == pages[0].url
+      end
+      it "should be able to find urls linking to a url" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+        page_store = Anemone.crawl(pages[0].url, @opts).pages
+        page_store.should respond_to(:pages_linking_to)
+        page_store.urls_linking_to(pages[2].url).size.should == 0
+        links_to_1 = page_store.urls_linking_to(pages[1].url)
+        links_to_1.size.should == 1
+        links_to_1.first.to_s.should == pages[0].url
+      end
+    end
+    describe Hash do
+      it_should_behave_like "page storage"
+      before(:all) do
+        @opts = {}
+      end
+    end
+    describe Storage::PStore do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @test_file = 'test.pstore'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => Storage.PStore(@test_file)}
+      end
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+    describe Storage::TokyoCabinet do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @test_file = 'test.tch'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.TokyoCabinet(@test_file)}
+      end
+      after(:each) do
+        @store.close
+      end
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+    describe Storage::SQLite3 do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @test_file = 'test.db'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+      end
+      after(:each) do
+        @store.close
+      end
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+    describe Storage::MongoDB do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @opts = {:storage => @store = Storage.MongoDB}
+      end
+      after(:each) do
+        @store.close
+      end
+    end
+    describe Storage::Redis do
+      it_should_behave_like "page storage"
+      before(:each) do
+        @opts = {:storage => @store = Storage.Redis}
+      end
+      after(:each) do
+        @store.close
+      end
+    end
+  end
+end