sutch-anemone 0.7.2

data/lib/anemone/storage/tokyo_cabinet.rb

@@ -0,0 +1,60 @@
+begin
+  require 'tokyocabinet'
+rescue LoadError
+  puts $!
+  puts "You need the tokyocabinet gem to use Anemone::Storage::TokyoCabinet"
+  exit
+end
+
+require 'forwardable'
+
+module Anemone
+  module Storage
+    class TokyoCabinet
+      extend Forwardable
+
+      def_delegators :@db, :close, :size, :keys, :has_key?
+
+      def initialize(file)
+        raise "TokyoCabinet filename must have .tch extension" if File.extname(file) != '.tch'
+        @db = ::TokyoCabinet::HDB::new
+        @db.open(file, ::TokyoCabinet::HDB::OWRITER | ::TokyoCabinet::HDB::OCREAT)
+        @db.clear
+      end
+
+      def [](key)
+        if value = @db[key]
+          load_value(value)
+        end
+      end
+
+      def []=(key, value)
+        @db[key] = [Marshal.dump(value)].pack("m")
+      end
+
+      def delete(key)
+        value = self[key]
+        @db.delete(key)
+        value
+      end
+
+      def each
+        @db.keys.each do |k|
+          yield(k, self[k])
+        end
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      private
+
+      def load_value(value)
+        Marshal.load(value.unpack("m")[0])
+      end
+
+    end
+  end
+end

data/lib/anemone/tentacle.rb

@@ -0,0 +1,39 @@
+require 'anemone/http'
+
+module Anemone
+  class Tentacle
+
+    #
+    # Create a new Tentacle
+    #
+    def initialize(link_queue, page_queue, opts = {})
+      @link_queue = link_queue
+      @page_queue = page_queue
+      @http = Anemone::HTTP.new(opts)
+      @opts = opts
+    end
+
+    #
+    # Gets links from @link_queue, and returns the fetched
+    # Page objects into @page_queue
+    #
+    def run
+      loop do
+        link, referer, depth = @link_queue.deq
+
+        break if link == :END
+
+        @http.fetch_pages(link, referer, depth).each { |page| @page_queue << page }
+
+        delay
+      end
+    end
+
+    private
+
+    def delay
+      sleep @opts[:delay] if @opts[:delay] > 0
+    end
+
+  end
+end

data/spec/anemone_spec.rb

@@ -0,0 +1,16 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+
+describe Anemone do
+
+  it "should have a version" do
+    Anemone.const_defined?('VERSION').should == true
+  end
+
+  it "should return a Anemone::Core from the crawl, which has a PageStore" do
+    result = Anemone.crawl(SPEC_DOMAIN)
+    result.should be_an_instance_of(Anemone::Core)
+    result.pages.should be_an_instance_of(Anemone::PageStore)
+  end
+
+end

data/spec/cookie_store_spec.rb

@@ -0,0 +1,28 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+
+module Anemone
+  describe CookieStore do
+
+    it "should start out empty if no cookies are specified" do
+      CookieStore.new.empty?.should be true
+    end
+
+    it "should accept a Hash of cookies in the constructor" do
+      CookieStore.new({'test' => 'cookie'})['test'].value.should == 'cookie'
+    end
+
+    it "should be able to merge an HTTP cookie string" do
+      cs = CookieStore.new({'a' => 'a', 'b' => 'b'})
+      cs.merge! "a=A; path=/, c=C; path=/"
+      cs['a'].value.should == 'A'
+      cs['b'].value.should == 'b'
+      cs['c'].value.should == 'C'
+    end
+
+    it "should have a to_s method to turn the cookies into a string for the HTTP Cookie header" do
+      CookieStore.new({'a' => 'a', 'b' => 'b'}).to_s.should == 'a=a;b=b'
+    end
+
+  end
+end

data/spec/core_spec.rb

@@ -0,0 +1,344 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+%w[pstore tokyo_cabinet sqlite3].each { |file| require "anemone/storage/#{file}.rb" }
+
+module Anemone
+  describe Core do
+
+    before(:each) do
+      FakeWeb.clean_registry
+    end
+
+    shared_examples_for "crawl" do
+      it "should crawl all the html pages in a domain by following <a> href's" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1', '2'])
+        pages << FakePage.new('1', :links => ['3'])
+        pages << FakePage.new('2')
+        pages << FakePage.new('3')
+
+        Anemone.crawl(pages[0].url, @opts).should have(4).pages
+      end
+
+      it "should not follow links that leave the original domain" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'], :hrefs => 'http://www.other.com/')
+        pages << FakePage.new('1')
+
+        core = Anemone.crawl(pages[0].url, @opts)
+
+        core.should have(2).pages
+        core.pages.keys.should_not include('http://www.other.com/')
+      end
+
+      it "should not follow redirects that leave the original domain" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'], :redirect => 'http://www.other.com/')
+        pages << FakePage.new('1')
+
+        core = Anemone.crawl(pages[0].url, @opts)
+
+        core.should have(2).pages
+        core.pages.keys.should_not include('http://www.other.com/')
+      end
+
+      it "should follow http redirects" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1', :redirect => '2')
+        pages << FakePage.new('2')
+
+        Anemone.crawl(pages[0].url, @opts).should have(3).pages
+      end
+
+      it "should follow with HTTP basic authentication" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1', '2'], :auth => true)
+        pages << FakePage.new('1', :links => ['3'], :auth => true)
+
+        Anemone.crawl(pages.first.auth_url, @opts).should have(3).pages
+      end
+
+      it "should accept multiple starting URLs" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1'])
+        pages << FakePage.new('1')
+        pages << FakePage.new('2', :links => ['3'])
+        pages << FakePage.new('3')
+
+        Anemone.crawl([pages[0].url, pages[2].url], @opts).should have(4).pages
+      end
+
+      it "should include the query string when following links" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1?foo=1'])
+        pages << FakePage.new('1?foo=1')
+        pages << FakePage.new('1')
+
+        core = Anemone.crawl(pages[0].url, @opts)
+
+        core.should have(2).pages
+        core.pages.keys.should_not include(pages[2].url)
+      end
+
+      it "should be able to skip links with query strings" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1?foo=1', '2'])
+        pages << FakePage.new('1?foo=1')
+        pages << FakePage.new('2')
+        
+        core = Anemone.crawl(pages[0].url, @opts) do |a|
+          a.skip_query_strings = true
+        end
+        
+        core.should have(2).pages
+      end
+
+      it "should be able to skip links based on a RegEx" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1', '2'])
+        pages << FakePage.new('1')
+        pages << FakePage.new('2')
+        pages << FakePage.new('3')
+
+        core = Anemone.crawl(pages[0].url, @opts) do |a|
+          a.skip_links_like /1/, /3/
+        end
+
+        core.should have(2).pages
+        core.pages.keys.should_not include(pages[1].url)
+        core.pages.keys.should_not include(pages[3].url)
+      end
+
+      it "should be able to call a block on every page" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1', '2'])
+        pages << FakePage.new('1')
+        pages << FakePage.new('2')
+
+        count = 0
+        Anemone.crawl(pages[0].url, @opts) do |a|
+          a.on_every_page { count += 1 }
+        end
+
+        count.should == 3
+      end
+
+      it "should not discard page bodies by default" do
+        Anemone.crawl(FakePage.new('0').url, @opts).pages.values#.first.doc.should_not be_nil
+      end
+
+      it "should optionally discard page bodies to conserve memory" do
+       # core = Anemone.crawl(FakePage.new('0').url, @opts.merge({:discard_page_bodies => true}))
+       # core.pages.values.first.doc.should be_nil
+      end
+
+      it "should provide a focus_crawl method to select the links on each page to follow" do
+        pages = []
+        pages << FakePage.new('0', :links => ['1', '2'])
+        pages << FakePage.new('1')
+        pages << FakePage.new('2')
+
+        core = Anemone.crawl(pages[0].url, @opts) do |a|
+          a.focus_crawl {|p| p.links.reject{|l| l.to_s =~ /1/}}
+        end
+
+        core.should have(2).pages
+        core.pages.keys.should_not include(pages[1].url)
+      end
+
+      it "should optionally delay between page requests" do
+        delay = 0.25
+
+        pages = []
+        pages << FakePage.new('0', :links => '1')
+        pages << FakePage.new('1')
+
+        start = Time.now
+        Anemone.crawl(pages[0].url, @opts.merge({:delay => delay}))
+        finish = Time.now
+
+        (finish - start).should satisfy {|t| t > delay * 2}
+      end
+
+      it "should optionally obey the robots exclusion protocol" do
+        pages = []
+        pages << FakePage.new('0', :links => '1')
+        pages << FakePage.new('1')
+        pages << FakePage.new('robots.txt',
+                              :body => "User-agent: *\nDisallow: /1",
+                              :content_type => 'text/plain')
+
+        core = Anemone.crawl(pages[0].url, @opts.merge({:obey_robots_txt => true}))
+        urls = core.pages.keys
+
+        urls.should include(pages[0].url)
+        urls.should_not include(pages[1].url)
+      end
+
+      it "should be able to set cookies to send with HTTP requests" do
+        cookies = {:a => '1', :b => '2'}
+        core = Anemone.crawl(FakePage.new('0').url) do |anemone|
+          anemone.cookies = cookies
+        end
+        core.opts[:cookies].should == cookies
+      end
+
+      it "should freeze the options once the crawl begins" do
+        core = Anemone.crawl(FakePage.new('0').url) do |anemone|
+          anemone.threads = 4
+          anemone.on_every_page do
+            lambda {anemone.threads = 2}.should raise_error
+          end
+        end
+        core.opts[:threads].should == 4
+      end
+
+      describe "many pages" do
+        before(:each) do
+          @pages, size = [], 5
+
+          size.times do |n|
+            # register this page with a link to the next page
+            link = (n + 1).to_s if n + 1 < size
+            @pages << FakePage.new(n.to_s, :links => Array(link))
+          end
+        end
+
+        it "should track the page depth and referer" do
+          core = Anemone.crawl(@pages[0].url, @opts)
+          previous_page = nil
+
+          @pages.each_with_index do |page, i|
+            page = core.pages[page.url]
+            page.should be
+            page.depth.should == i
+
+            if previous_page
+              page.referer.should == previous_page.url
+            else
+              page.referer.should be_nil
+            end
+            previous_page = page
+          end
+        end
+
+        it "should optionally limit the depth of the crawl" do
+          core = Anemone.crawl(@pages[0].url, @opts.merge({:depth_limit => 3}))
+          core.should have(4).pages
+        end
+      end
+
+    end
+
+    describe Hash do
+      it_should_behave_like "crawl"
+
+      before(:all) do
+        @opts = {}
+      end
+    end
+
+    describe Storage::PStore do
+      it_should_behave_like "crawl"
+
+      before(:all) do
+        @test_file = 'test.pstore'
+      end
+
+      before(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => Storage.PStore(@test_file)}
+      end
+
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
+    describe Storage::TokyoCabinet do
+      it_should_behave_like "crawl"
+
+      before(:all) do
+        @test_file = 'test.tch'
+      end
+
+      before(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.TokyoCabinet(@test_file)}
+      end
+
+      after(:each) do
+        @store.close
+      end
+
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
+    describe Storage::SQLite3 do
+      it_should_behave_like "crawl"
+
+      before(:all) do
+        @test_file = 'test.db'
+      end
+
+      before(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+      end
+
+      after(:each) do
+        @store.close
+      end
+
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
+    describe "options" do
+      it "should accept options for the crawl" do
+        core = Anemone.crawl(SPEC_DOMAIN, :verbose => false,
+                                          :threads => 2,
+                                          :discard_page_bodies => true,
+                                          :user_agent => 'test',
+                                          :obey_robots_txt => true,
+                                          :depth_limit => 3)
+
+        core.opts[:verbose].should == false
+        core.opts[:threads].should == 2
+        core.opts[:discard_page_bodies].should == true
+        core.opts[:delay].should == 0
+        core.opts[:user_agent].should == 'test'
+        core.opts[:obey_robots_txt].should == true
+        core.opts[:depth_limit].should == 3
+      end
+
+      it "should accept options via setter methods in the crawl block" do
+        core = Anemone.crawl(SPEC_DOMAIN) do |a|
+          a.verbose = false
+          a.threads = 2
+          a.discard_page_bodies = true
+          a.user_agent = 'test'
+          a.obey_robots_txt = true
+          a.depth_limit = 3
+        end
+
+        core.opts[:verbose].should == false
+        core.opts[:threads].should == 2
+        core.opts[:discard_page_bodies].should == true
+        core.opts[:delay].should == 0
+        core.opts[:user_agent].should == 'test'
+        core.opts[:obey_robots_txt].should == true
+        core.opts[:depth_limit].should == 3
+      end
+
+      it "should use 1 thread if a delay is requested" do
+        Anemone.crawl(SPEC_DOMAIN, :delay => 0.01, :threads => 2).opts[:threads].should == 1
+      end
+    end
+
+  end
+end