spidr 0.2.7 → 0.3.0

data/spec/page_spec.rb

@@ -100,13 +100,6 @@ describe Page do
     end
 
     it "should provide access to the raw Cookie" do
-      cookie = @page.raw_cookie
-
-      cookie.should_not be_nil
-      cookie.should_not be_empty
-    end
-
-    it "should still support the deprecated #cookie method" do
       cookie = @page.cookie
 
       cookie.should_not be_nil

data/spec/rules_spec.rb

@@ -3,39 +3,41 @@ require 'spidr/rules'
 require 'spec_helper'
 
 describe Rules do
+  subject { Rules }
+
   it "should accept data based on acceptance data" do
-    rules = Rules.new(:accept => [1])
+    rules = subject.new(:accept => [1])
 
     rules.accept?(1).should == true
   end
 
   it "should accept data based on acceptance regexps" do
-    rules = Rules.new(:accept => [/1/])
+    rules = subject.new(:accept => [/1/])
 
     rules.accept?('1').should == true
   end
 
   it "should match non-Strings using acceptance regexps" do
-    rules = Rules.new(:accept => [/1/])
+    rules = subject.new(:accept => [/1/])
 
     rules.accept?(1).should == true
   end
 
   it "should accept data using acceptance lambdas" do
-    rules = Rules.new(:accept => [lambda { |data| data > 2 }])
+    rules = subject.new(:accept => [lambda { |data| data > 2 }])
 
     rules.accept?(3).should == true
   end
 
   it "should reject data that does not match any acceptance patterns" do
-    rules = Rules.new(:accept => [1, 2, 3])
+    rules = subject.new(:accept => [1, 2, 3])
 
     rules.accept?(2).should == true
     rules.accept?(4).should == false
   end
 
   it "should accept data that does not match any rejection patterns" do
-    rules = Rules.new(:reject => [1, 2, 3])
+    rules = subject.new(:reject => [1, 2, 3])
 
     rules.accept?(2).should == false
     rules.accept?(4).should == true

data/spec/sanitizers_spec.rb

@@ -5,61 +5,55 @@ require 'spec_helper'
 
 describe Sanitizers do
   describe "sanitize_url" do
-    before(:all) do
-      @agent = Agent.new
-      @url = 'http://host.com'
-    end
+    let(:url) { 'http://host.com' }
+    before(:all) { @agent = Agent.new }
 
     it "should sanitize URLs" do
       agent = Agent.new
-      clean_url = agent.sanitize_url(URI(@url))
+      clean_url = agent.sanitize_url(URI(url))
 
       clean_url.host.should == 'host.com'
     end
 
     it "should sanitize URLs given as Strings" do
       agent = Agent.new
-      clean_url = agent.sanitize_url(@url)
+      clean_url = agent.sanitize_url(url)
 
       clean_url.host.should == 'host.com'
     end
   end
 
   describe "strip_fragments" do
-    before(:all) do
-      @url = URI("http://host.com/page#lol")
-    end
+    let(:url) { URI("http://host.com/page#lol") }
 
     it "should strip fragment components by default" do
       agent = Agent.new
-      clean_url = agent.sanitize_url(@url)
+      clean_url = agent.sanitize_url(url)
 
       clean_url.fragment.should be_nil
     end
 
     it "should allow perserving fragment components" do
       agent = Agent.new(:strip_fragments => false)
-      clean_url = agent.sanitize_url(@url)
+      clean_url = agent.sanitize_url(url)
 
       clean_url.fragment.should == 'lol'
     end
   end
 
   describe "strip_query" do
-    before(:all) do
-      @url = URI("http://host.com/page?x=1")
-    end
+    let(:url) { URI("http://host.com/page?x=1") }
 
     it "should not strip query components by default" do
       agent = Agent.new
-      clean_url = agent.sanitize_url(@url)
+      clean_url = agent.sanitize_url(url)
 
       clean_url.query.should == 'x=1'
     end
 
     it "should allow stripping of query components" do
       agent = Agent.new(:strip_query => true)
-      clean_url = agent.sanitize_url(@url)
+      clean_url = agent.sanitize_url(url)
 
       clean_url.query.should be_nil
     end

data/spec/spec_helper.rb

@@ -1,15 +1,4 @@
-require 'rubygems'
-require 'bundler'
-
-begin
-  Bundler.setup(:runtime, :test)
-rescue Bundler::BundlerError => e
-  STDERR.puts e.message
-  STDERR.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-
-require 'spec'
+require 'rspec'
 require 'spidr/version'
 
 include Spidr

data/spec/spidr_spec.rb

@@ -4,36 +4,36 @@ require 'spec_helper'
 
 describe Spidr do
   it "should have a VERSION constant" do
-    Spidr.const_defined?('VERSION').should == true
+    subject.const_defined?('VERSION').should == true
   end
 
   describe "proxy" do
     after(:all) do
-      Spidr.disable_proxy!
+      subject.disable_proxy!
     end
 
     it "should not have proxy settings by default" do
-      Spidr.proxy[:host].should be_nil
+      subject.proxy[:host].should be_nil
     end
 
     it "should allow setting new proxy settings" do
-      Spidr.proxy = {:host => 'example.com', :port => 8010}
+      subject.proxy = {:host => 'example.com', :port => 8010}
 
-      Spidr.proxy[:host].should == 'example.com'
-      Spidr.proxy[:port].should == 8010
+      subject.proxy[:host].should == 'example.com'
+      subject.proxy[:port].should == 8010
     end
 
     it "should default the :port option of new proxy settings" do
-      Spidr.proxy = {:host => 'example.com'}
+      subject.proxy = {:host => 'example.com'}
 
-      Spidr.proxy[:host].should == 'example.com'
-      Spidr.proxy[:port].should == Spidr::COMMON_PROXY_PORT
+      subject.proxy[:host].should == 'example.com'
+      subject.proxy[:port].should == Spidr::COMMON_PROXY_PORT
     end
 
     it "should allow disabling the proxy" do
-      Spidr.disable_proxy!
+      subject.disable_proxy!
 
-      Spidr.proxy[:host].should be_nil
+      subject.proxy[:host].should be_nil
     end
   end
 end

data/spidr.gemspec

@@ -1,114 +1,15 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
 
-Gem::Specification.new do |s|
-  s.name = %q{spidr}
-  s.version = "0.2.7"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Postmodern"]
-  s.date = %q{2010-08-17}
-  s.description = %q{Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.}
-  s.email = %q{postmodern.mod3@gmail.com}
-  s.extra_rdoc_files = [
-    "ChangeLog.md",
-    "LICENSE.txt",
-    "README.md"
-  ]
-  s.files = [
-    ".gitignore",
-    ".specopts",
-    ".yardopts",
-    "ChangeLog.md",
-    "Gemfile",
-    "Gemfile.lock",
-    "LICENSE.txt",
-    "README.md",
-    "Rakefile",
-    "lib/spidr.rb",
-    "lib/spidr/actions.rb",
-    "lib/spidr/actions/actions.rb",
-    "lib/spidr/actions/exceptions.rb",
-    "lib/spidr/actions/exceptions/action.rb",
-    "lib/spidr/actions/exceptions/paused.rb",
-    "lib/spidr/actions/exceptions/skip_link.rb",
-    "lib/spidr/actions/exceptions/skip_page.rb",
-    "lib/spidr/agent.rb",
-    "lib/spidr/auth_credential.rb",
-    "lib/spidr/auth_store.rb",
-    "lib/spidr/cookie_jar.rb",
-    "lib/spidr/events.rb",
-    "lib/spidr/extensions.rb",
-    "lib/spidr/extensions/uri.rb",
-    "lib/spidr/filters.rb",
-    "lib/spidr/page.rb",
-    "lib/spidr/rules.rb",
-    "lib/spidr/sanitizers.rb",
-    "lib/spidr/session_cache.rb",
-    "lib/spidr/spidr.rb",
-    "lib/spidr/version.rb",
-    "spec/actions_spec.rb",
-    "spec/agent_spec.rb",
-    "spec/auth_store_spec.rb",
-    "spec/cookie_jar_spec.rb",
-    "spec/extensions/uri_spec.rb",
-    "spec/filters_spec.rb",
-    "spec/helpers/history.rb",
-    "spec/helpers/page.rb",
-    "spec/helpers/wsoc.rb",
-    "spec/page_examples.rb",
-    "spec/page_spec.rb",
-    "spec/rules_spec.rb",
-    "spec/sanitizers_spec.rb",
-    "spec/session_cache.rb",
-    "spec/spec_helper.rb",
-    "spec/spidr_spec.rb",
-    "spidr.gemspec"
-  ]
-  s.has_rdoc = %q{yard}
-  s.homepage = %q{http://github.com/postmodern/spidr}
-  s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{A versatile Ruby web spidering library}
-  s.test_files = [
-    "spec/actions_spec.rb",
-    "spec/agent_spec.rb",
-    "spec/auth_store_spec.rb",
-    "spec/cookie_jar_spec.rb",
-    "spec/extensions/uri_spec.rb",
-    "spec/filters_spec.rb",
-    "spec/helpers/history.rb",
-    "spec/helpers/page.rb",
-    "spec/helpers/wsoc.rb",
-    "spec/page_examples.rb",
-    "spec/page_spec.rb",
-    "spec/rules_spec.rb",
-    "spec/sanitizers_spec.rb",
-    "spec/session_cache.rb",
-    "spec/spec_helper.rb",
-    "spec/spidr_spec.rb"
-  ]
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<rake>, ["~> 0.8.7"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.4.0"])
-      s.add_development_dependency(%q<rspec>, ["~> 1.3.0"])
-    else
-      s.add_dependency(%q<rake>, ["~> 0.8.7"])
-      s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
-      s.add_dependency(%q<rspec>, ["~> 1.3.0"])
-    end
-  else
-    s.add_dependency(%q<rake>, ["~> 0.8.7"])
-    s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
-    s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+begin
+  Ore::Specification.new do |gemspec|
+    # custom logic here
+  end
+rescue NameError
+  begin
+    require 'ore/specification'
+    retry
+  rescue LoadError
+    STDERR.puts "The '#{__FILE__}' file requires Ore."
+    STDERR.puts "Run `gem install ore-core` to install Ore."
   end
 end
-

metadata

@@ -1,12 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: spidr
 version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 2
-  - 7
-  version: 0.2.7
+  prerelease: 
+  version: 0.3.0
 platform: ruby
 authors: 
 - Postmodern
@@ -14,74 +10,59 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-17 00:00:00 -07:00
-default_executable: 
+date: 2011-04-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: nokogiri
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 8
-        - 7
-        version: 0.8.7
-  type: :development
+        version: "1.3"
+  type: :runtime
   prerelease: false
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: jeweler
+  name: bundler
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 4
-        - 0
-        version: 1.4.0
+        version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: yard
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 3
-        - 0
-        version: 1.3.0
+        version: 0.6.0
   type: :development
   prerelease: false
   version_requirements: *id003
 description: Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.
-email: postmodern.mod3@gmail.com
+email: 
+- postmodern.mod3@gmail.com
 executables: []
 
 extensions: []
 
 extra_rdoc_files: 
-- ChangeLog.md
-- LICENSE.txt
 - README.md
 files: 
-- .gitignore
-- .specopts
+- .rspec
 - .yardopts
 - ChangeLog.md
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
+- gemspec.yml
 - lib/spidr.rb
 - lib/spidr/actions.rb
 - lib/spidr/actions/actions.rb
@@ -93,11 +74,14 @@ files:
 - lib/spidr/agent.rb
 - lib/spidr/auth_credential.rb
 - lib/spidr/auth_store.rb
+- lib/spidr/body.rb
 - lib/spidr/cookie_jar.rb
 - lib/spidr/events.rb
 - lib/spidr/extensions.rb
 - lib/spidr/extensions/uri.rb
 - lib/spidr/filters.rb
+- lib/spidr/headers.rb
+- lib/spidr/links.rb
 - lib/spidr/page.rb
 - lib/spidr/rules.rb
 - lib/spidr/sanitizers.rb
@@ -121,7 +105,6 @@ files:
 - spec/spec_helper.rb
 - spec/spidr_spec.rb
 - spidr.gemspec
-has_rdoc: yard
 homepage: http://github.com/postmodern/spidr
 licenses: 
 - MIT
@@ -135,39 +118,28 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 4533863298463290280
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+      version: 1.3.6
 requirements: []
 
-rubyforge_project: 
-rubygems_version: 1.3.7
+rubyforge_project: spidr
+rubygems_version: 1.7.2
 signing_key: 
 specification_version: 3
 summary: A versatile Ruby web spidering library
 test_files: 
-- spec/actions_spec.rb
 - spec/agent_spec.rb
+- spec/actions_spec.rb
+- spec/rules_spec.rb
+- spec/extensions/uri_spec.rb
 - spec/auth_store_spec.rb
 - spec/cookie_jar_spec.rb
-- spec/extensions/uri_spec.rb
 - spec/filters_spec.rb
-- spec/helpers/history.rb
-- spec/helpers/page.rb
-- spec/helpers/wsoc.rb
-- spec/page_examples.rb
-- spec/page_spec.rb
-- spec/rules_spec.rb
 - spec/sanitizers_spec.rb
-- spec/session_cache.rb
-- spec/spec_helper.rb
 - spec/spidr_spec.rb
+- spec/page_spec.rb