sutch-anemone 0.7.2

data/spec/resource_spec.rb

@@ -0,0 +1,91 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+
+# Based on Anemone::Page (i.e., only tests HTML resources)
+
+module Anemone
+  describe Resource do
+
+    before(:each) do
+      FakeWeb.clean_registry
+      @http = Anemone::HTTP.new(:page_class => Anemone::Page)
+
+      @page = @http.fetch_page(FakePage.new('home', :links => '1').url)
+    end
+
+    describe "#to_hash" do
+      it "converts the page to a hash" do
+        hash = @page.to_hash
+        hash['url'].should == @page.url.to_s
+        hash['referer'].should == @page.referer.to_s
+        hash['links'].should == @page.links.map(&:to_s)
+      end
+    end
+
+    describe "#from_hash" do
+      it "converts from a hash to a Page" do
+        page = @page.dup
+        page.depth = 1
+        converted = Page.from_hash(page.to_hash)
+        converted.links.should == page.links
+        converted.depth.should == page.depth
+      end
+    end
+
+    describe "#links" do
+      it "should not convert anchors to %23" do
+        page = @http.fetch_page(FakePage.new('', :body => '<a href="#top">Top</a>').url)
+        page.links.should have(1).link
+        page.links.first.to_s.should == SPEC_DOMAIN
+      end
+    end
+
+    it "should detect, store and expose the base url for the page head" do
+      base = "#{SPEC_DOMAIN}path/to/base_url/"
+      page = @http.fetch_page(FakePage.new('body_test', {:base => base}).url)
+      page.base.should == URI(base)
+      @page.base.should be_nil
+    end
+
+    it "should have a method to convert a relative url to an absolute one" do
+      @page.should respond_to(:to_absolute)
+
+      # Identity
+      @page.to_absolute(@page.url).should == @page.url
+      @page.to_absolute("").should == @page.url
+
+      # Root-ness
+      @page.to_absolute("/").should == URI("#{SPEC_DOMAIN}")
+
+      # Relativeness
+      relative_path = "a/relative/path"
+      @page.to_absolute(relative_path).should == URI("#{SPEC_DOMAIN}#{relative_path}")
+
+      deep_page = @http.fetch_page(FakePage.new('home/deep', :links => '1').url)
+      upward_relative_path = "../a/relative/path"
+      deep_page.to_absolute(upward_relative_path).should == URI("#{SPEC_DOMAIN}#{relative_path}")
+
+      # The base URL case
+      base_path = "path/to/base_url/"
+      base = "#{SPEC_DOMAIN}#{base_path}"
+      page = @http.fetch_page(FakePage.new('home', {:base => base}).url)
+
+      # Identity
+      page.to_absolute(page.url).should == page.url
+      # It should revert to the base url
+      page.to_absolute("").should_not == page.url
+
+      # Root-ness
+      page.to_absolute("/").should == URI("#{SPEC_DOMAIN}")
+
+      # Relativeness
+      relative_path = "a/relative/path"
+      page.to_absolute(relative_path).should == URI("#{base}#{relative_path}")
+
+      upward_relative_path = "../a/relative/path"
+      upward_base = "#{SPEC_DOMAIN}path/to/"
+      page.to_absolute(upward_relative_path).should == URI("#{upward_base}#{relative_path}")
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'fakeweb'
+require File.dirname(__FILE__) + '/fakeweb_helper'
+
+$:.unshift(File.dirname(__FILE__) + '/../lib/')
+require 'anemone'
+
+SPEC_DOMAIN = 'http://www.example.com/'

data/spec/storage_spec.rb

@@ -0,0 +1,252 @@
+$:.unshift(File.dirname(__FILE__))
+require 'spec_helper'
+
+%w[pstore tokyo_cabinet kyoto_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+
+module Anemone
+  describe Storage do
+
+    describe ".Hash" do
+      it "returns a Hash adapter" do
+        Anemone::Storage.Hash.should be_an_instance_of(Hash)
+      end
+    end
+
+    describe ".PStore" do
+      it "returns a PStore adapter" do
+        test_file = 'test.pstore'
+        Anemone::Storage.PStore(test_file).should be_an_instance_of(Anemone::Storage::PStore)
+      end
+    end
+
+    describe ".TokyoCabinet" do
+      it "returns a TokyoCabinet adapter" do
+        test_file = 'test.tch'
+        store = Anemone::Storage.TokyoCabinet(test_file)
+        store.should be_an_instance_of(Anemone::Storage::TokyoCabinet)
+        store.close
+      end
+    end
+
+    describe ".KyotoCabinet" do
+      context "when the file is specified" do
+        it "returns a KyotoCabinet adapter using that file" do
+          test_file = 'test.kch'
+          store = Anemone::Storage.KyotoCabinet(test_file)
+          store.should be_an_instance_of(Anemone::Storage::KyotoCabinet)
+          store.close
+        end
+      end
+
+      context "when no file is specified" do
+        it "returns a KyotoCabinet adapter using the default filename" do
+          store = Anemone::Storage.KyotoCabinet
+          store.should be_an_instance_of(Anemone::Storage::KyotoCabinet)
+          store.close
+        end
+      end
+    end
+
+    describe ".SQLite3" do
+      it "returns a SQLite3 adapter" do
+        test_file = 'test.db'
+        store = Anemone::Storage.SQLite3(test_file)
+        store.should be_an_instance_of(Anemone::Storage::SQLite3)
+        store.close
+      end
+    end
+
+    describe ".MongoDB" do
+      it "returns a MongoDB adapter" do
+        store = Anemone::Storage.MongoDB
+        store.should be_an_instance_of(Anemone::Storage::MongoDB)
+        store.close
+      end
+    end
+
+    describe ".MongoDB" do
+      it "returns a Redis adapter" do
+        store = Anemone::Storage.Redis
+        store.should be_an_instance_of(Anemone::Storage::Redis)
+        store.close
+      end
+    end
+
+    module Storage
+      shared_examples_for "storage engine" do
+
+        before(:each) do
+          @url = SPEC_DOMAIN
+          @page = Page.new(URI(@url))
+        end
+
+        it "should implement [] and []=" do
+          @store.should respond_to(:[])
+          @store.should respond_to(:[]=)
+
+          @store[@url] = @page 
+          @store[@url].url.should == URI(@url)
+        end
+
+        it "should implement has_key?" do
+          @store.should respond_to(:has_key?)
+
+          @store[@url] = @page
+          @store.has_key?(@url).should == true
+
+          @store.has_key?('missing').should == false
+        end
+
+        it "should implement delete" do
+          @store.should respond_to(:delete)
+
+          @store[@url] = @page
+          @store.delete(@url).url.should == @page.url
+          @store.has_key?(@url).should  == false
+        end
+
+        it "should implement keys" do
+          @store.should respond_to(:keys)
+
+          urls = [SPEC_DOMAIN, SPEC_DOMAIN + 'test', SPEC_DOMAIN + 'another']
+          pages = urls.map { |url| Page.new(URI(url)) }
+          urls.zip(pages).each { |arr| @store[arr[0]] = arr[1] }
+
+          (@store.keys - urls).should == [] 
+        end
+
+        it "should implement each" do
+          @store.should respond_to(:each)
+
+          urls = [SPEC_DOMAIN, SPEC_DOMAIN + 'test', SPEC_DOMAIN + 'another']
+          pages = urls.map { |url| Page.new(URI(url)) }
+          urls.zip(pages).each { |arr| @store[arr[0]] = arr[1] }
+
+          result = {}
+          @store.each { |k, v| result[k] = v }
+          (result.keys - urls).should == [] 
+          (result.values.map { |page| page.url.to_s } - urls).should == []
+        end
+
+        it "should implement merge!, and return self" do
+          @store.should respond_to(:merge!)
+
+          hash = {SPEC_DOMAIN => Page.new(URI(SPEC_DOMAIN)),
+                  SPEC_DOMAIN + 'test' => Page.new(URI(SPEC_DOMAIN + 'test'))}
+          merged = @store.merge! hash
+          hash.each { |key, value| @store[key].url.to_s.should == key }
+
+          merged.should === @store
+        end
+
+        it "should correctly deserialize nil redirect_to when loading" do
+          @page.redirect_to.should be_nil
+          @store[@url] = @page 
+          @store[@url].redirect_to.should be_nil
+        end
+      end
+
+      describe PStore do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.pstore'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.PStore(@test_file)
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+      end
+
+      describe TokyoCabinet do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.tch'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.TokyoCabinet(@test_file)
+        end
+
+        after(:each) do
+          @store.close
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+
+        it "should raise an error if supplied with a file extension other than .tch" do
+          lambda { Anemone::Storage.TokyoCabinet('test.tmp') }.should raise_error(RuntimeError)
+        end
+      end
+
+      describe KyotoCabinet do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.kch'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.KyotoCabinet(@test_file)
+        end
+
+        after(:each) do
+          @store.close
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+
+        it "should raise an error if supplied with a file extension other than .kch" do
+          lambda { Anemone::Storage.KyotoCabinet('test.tmp') }.should raise_error(RuntimeError)
+        end
+      end
+
+      describe SQLite3 do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.db'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.SQLite3(@test_file)
+        end
+
+        after(:each) do
+          @store.close
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+
+      end
+
+      describe Storage::MongoDB do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @store = Storage.MongoDB
+        end
+
+        after(:each) do
+          @store.close
+        end
+      end
+
+      describe Storage::Redis do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @store = Storage.Redis
+        end
+
+        after(:each) do
+          @store.close
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,281 @@
+--- !ruby/object:Gem::Specification
+name: sutch-anemone
+version: !ruby/object:Gem::Version
+  version: 0.7.2
+platform: ruby
+authors:
+- Chris Kite (Dennis Sutch's fork)
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: robotex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: content_urls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.8
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: tokyocabinet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.29'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.29'
+- !ruby/object:Gem::Dependency
+  name: kyotocabinet-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.27.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.27.1
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+description: 
+email: 
+executables:
+- anemone
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+files:
+- VERSION
+- LICENSE.txt
+- CHANGELOG.rdoc
+- README.rdoc
+- Rakefile
+- lib/anemone/cli/count.rb
+- lib/anemone/cli/cron.rb
+- lib/anemone/cli/pagedepth.rb
+- lib/anemone/cli/serialize.rb
+- lib/anemone/cli/url_list.rb
+- lib/anemone/cli.rb
+- lib/anemone/cookie_store.rb
+- lib/anemone/core.rb
+- lib/anemone/exceptions.rb
+- lib/anemone/http.rb
+- lib/anemone/page.rb
+- lib/anemone/page_store.rb
+- lib/anemone/resource.rb
+- lib/anemone/storage/base.rb
+- lib/anemone/storage/exceptions.rb
+- lib/anemone/storage/kyoto_cabinet.rb
+- lib/anemone/storage/mongodb.rb
+- lib/anemone/storage/pstore.rb
+- lib/anemone/storage/redis.rb
+- lib/anemone/storage/sqlite3.rb
+- lib/anemone/storage/tokyo_cabinet.rb
+- lib/anemone/storage.rb
+- lib/anemone/tentacle.rb
+- lib/anemone.rb
+- spec/anemone_spec.rb
+- spec/cookie_store_spec.rb
+- spec/core_spec.rb
+- spec/fakeweb_helper.rb
+- spec/http_spec.rb
+- spec/page_spec.rb
+- spec/page_store_spec.rb
+- spec/resource_spec.rb
+- spec/spec_helper.rb
+- spec/storage_spec.rb
+- bin/anemone
+homepage: http://anemone.rubyforge.org
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options:
+- -m
+- README.rdoc
+- -t
+- Anemone
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: anemone
+rubygems_version: 2.0.6
+signing_key: 
+specification_version: 4
+summary: Anemone web-spider framework (sutch's fork)
+test_files:
+- spec/anemone_spec.rb
+- spec/cookie_store_spec.rb
+- spec/core_spec.rb
+- spec/fakeweb_helper.rb
+- spec/http_spec.rb
+- spec/page_spec.rb
+- spec/page_store_spec.rb
+- spec/resource_spec.rb
+- spec/spec_helper.rb
+- spec/storage_spec.rb
+has_rdoc: true