spidr 0.2.6 → 0.2.7

data/ChangeLog.md

@@ -1,3 +1,10 @@
+### 0.2.7 / 2010-08-17
+
+* Added {Spidr::CookieJar#cookies_for_host} (thanks zapnap).
+* Renamed `Spidr::Page#cookie` to {Spidr::Page#raw_cookie}.
+* Rescue `URI::InvalidComponentError` exceptions in
+  {Spidr::Page#to_absolute} (thanks zapnap).
+
 ### 0.2.6 / 2010-07-05
 
 * Fixed a bug in {Spidr::Page#meta_redirect}, by calling

data/Gemfile.lock

@@ -0,0 +1,39 @@
+GIT
+  remote: git://github.com/technicalpickles/jeweler.git
+  revision: 7856803
+  specs:
+    jeweler (1.4.0)
+      bundler (>= 0.9.5)
+      gemcutter (>= 0.1.0)
+      git (>= 1.2.5)
+      rake
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    gemcutter (0.6.1)
+    git (1.2.5)
+    json_pure (1.4.6)
+    nokogiri (1.4.3.1)
+    rack (1.2.1)
+    rake (0.8.7)
+    rdiscount (1.6.5)
+    rspec (1.3.0)
+    sinatra (1.0)
+      rack (>= 1.0)
+    wsoc (0.1.3)
+      json_pure (~> 1.4.0)
+      sinatra (~> 1.0)
+    yard (0.5.8)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jeweler (~> 1.4.0)!
+  nokogiri (>= 1.3.0)
+  rake (~> 0.8.7)
+  rdiscount (~> 1.6.3)
+  rspec (~> 1.3.0)
+  wsoc (~> 0.1.3)
+  yard (~> 0.5.3)

data/Rakefile

@@ -24,6 +24,7 @@ Jeweler::Tasks.new do |gem|
   gem.authors = ['Postmodern']
   gem.has_rdoc = 'yard'
 end
+Jeweler::GemcutterTasks.new
 
 require 'spec/rake/spectask'
 Spec::Rake::SpecTask.new(:spec) do |spec|

data/lib/spidr/cookie_jar.rb

@@ -122,7 +122,7 @@ module Spidr
       if @dirty.include?(host)
         values = []
 
-        @params[host].each do |name,value|
+        cookies_for_host(host).each do |name,value|
           values << "#{name}=#{value}"
         end
 
@@ -130,23 +130,40 @@ module Spidr
         @dirty.delete(host)
       end
 
-      hdomain = host.split('.')
+      return @cookies[host]
+    end
+
+    #
+    # Returns raw cookie value pairs for a given host. Includes cookies set on
+    # parent domain(s).
+    #
+    # @param [String] host
+    #   The name of the host.
+    #
+    # @return [Hash{String => String}]
+    #   Cookie params.
+    #
+    # @since 0.2.7
+    #
+    def cookies_for_host(host)
+      host_cookies = (@params[host] || {})
+      sub_domains = host.split('.')
 
-      if hdomain.length > 2
-        parent_cookies = for_host(hdomain[1..-1].join('.'))
+      while sub_domains.length > 2
+        sub_domains.shift
 
-        unless (parent_cookies.nil? || parent_cookies.empty?)
-          @cookies[host] = if @cookies[host].nil?
-                             # inherit the parent cookies
-                             parent_cookies
-                           else
-                             # merge the parent cookies with any host-specific cookies
-                             "#{parent_cookies}; #{@cookies[host]}"
-                           end
+        if (parent_cookies = @params[sub_domains.join('.')])
+          parent_cookies.each do |name,value|
+            # copy in the parent cookies, only if they haven't been
+            # overridden yet.
+            unless host_cookies.has_key?(name)
+              host_cookies[name] = value
+            end
+          end
         end
       end
 
-      return @cookies[host]
+      return host_cookies
     end
 
     # 

data/lib/spidr/page.rb

@@ -289,10 +289,29 @@ module Spidr
     # @return [String]
     #   The raw Cookie from the response.
     #
+    # @since 0.2.7
+    #
+    def raw_cookie
+      (@response['Set-Cookie'] || '')
+    end
+
+    #
+    # The raw Cookie String sent along with the page.
+    #
+    # @return [String]
+    #   The raw Cookie from the response.
+    #
+    # @deprecated
+    #   Deprecated in 0.2.7 and will be removed in 0.3.0.
+    #   Use {#raw_cookie} instead.
+    #
     # @since 0.2.2
     #
     def cookie
-      (@response['Set-Cookie'] || '')
+      STDERR.puts 'DEPRECATION: Spidr::Page#cookie will be removed in 0.3.0'
+      STDERR.puts 'DEPRECATION: Use Spidr::Page#raw_cookie instead'
+
+      return raw_cookie
     end
 
     #
@@ -507,7 +526,7 @@ module Spidr
     def to_absolute(link)
       begin
         url = @url.merge(link.to_s)
-      rescue URI::InvalidURIError
+      rescue URI::InvalidURIError, URI::InvalidComponentError
         return nil
       end
 

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.2.6'
+  VERSION = '0.2.7'
 end

data/spec/cookie_jar_spec.rb

@@ -77,6 +77,40 @@ describe CookieJar do
     end
   end
 
+  describe "cookies_for_host" do
+    before(:each) do
+      @cookie_jar = CookieJar.new
+    end
+
+    it "should return an empty Hash for unknown hosts" do
+      @cookie_jar.cookies_for_host('lol.com').should be_empty
+    end
+
+    it "should return an empty Hash for hosts with no cookie params" do
+      @cookie_jar['lol.com'] = {}
+
+      @cookie_jar.cookies_for_host('lol.com').should be_empty
+    end
+
+    it "should return cookie parameters for the host" do
+      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      @cookie_jar['zerosum.org'] = {'other' => '1'}
+      cookie = @cookie_jar.cookies_for_host('zerosum.org')
+
+      cookie['admin'].should == 'ofcourseiam'
+      cookie['other'].should == '1'
+    end
+
+    it "should include cookies for the parent domain" do
+      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      @cookie_jar['sub.zerosum.org'] = {'other' => '1'}
+      cookie = @cookie_jar.cookies_for_host('sub.zerosum.org')
+
+      cookie['admin'].should == 'ofcourseiam'
+      cookie['other'].should == '1'
+    end
+  end
+
   describe "for_host" do
     before(:each) do
       @cookie_jar = CookieJar.new

data/spec/page_spec.rb

@@ -100,6 +100,13 @@ describe Page do
     end
 
     it "should provide access to the raw Cookie" do
+      cookie = @page.raw_cookie
+
+      cookie.should_not be_nil
+      cookie.should_not be_empty
+    end
+
+    it "should still support the deprecated #cookie method" do
       cookie = @page.cookie
 
       cookie.should_not be_nil

data/spidr.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{spidr}
-  s.version = "0.2.6"
+  s.version = "0.2.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Postmodern"]
-  s.date = %q{2010-07-05}
+  s.date = %q{2010-08-17}
   s.description = %q{Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.}
   s.email = %q{postmodern.mod3@gmail.com}
   s.extra_rdoc_files = [
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
     ".yardopts",
     "ChangeLog.md",
     "Gemfile",
+    "Gemfile.lock",
     "LICENSE.txt",
     "README.md",
     "Rakefile",
@@ -96,18 +97,15 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<nokogiri>, [">= 1.3.0"])
       s.add_development_dependency(%q<rake>, ["~> 0.8.7"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.4.0"])
       s.add_development_dependency(%q<rspec>, ["~> 1.3.0"])
     else
-      s.add_dependency(%q<nokogiri>, [">= 1.3.0"])
       s.add_dependency(%q<rake>, ["~> 0.8.7"])
       s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
       s.add_dependency(%q<rspec>, ["~> 1.3.0"])
     end
   else
-    s.add_dependency(%q<nokogiri>, [">= 1.3.0"])
     s.add_dependency(%q<rake>, ["~> 0.8.7"])
     s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
     s.add_dependency(%q<rspec>, ["~> 1.3.0"])

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 6
-  version: 0.2.6
+  - 7
+  version: 0.2.7
 platform: ruby
 authors: 
 - Postmodern
@@ -14,27 +14,12 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-05 00:00:00 -07:00
+date: 2010-08-17 00:00:00 -07:00
 default_executable: 
 dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 3
-        - 0
-        version: 1.3.0
-  type: :runtime
-  prerelease: false
-  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: rake
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -46,10 +31,10 @@ dependencies:
         version: 0.8.7
   type: :development
   prerelease: false
-  version_requirements: *id002
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: jeweler
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -61,10 +46,10 @@ dependencies:
         version: 1.4.0
   type: :development
   prerelease: false
-  version_requirements: *id003
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: rspec
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -76,7 +61,7 @@ dependencies:
         version: 1.3.0
   type: :development
   prerelease: false
-  version_requirements: *id004
+  version_requirements: *id003
 description: Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.
 email: postmodern.mod3@gmail.com
 executables: []
@@ -93,6 +78,7 @@ files:
 - .yardopts
 - ChangeLog.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -149,7 +135,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -707506015
+      hash: 4533863298463290280
       segments: 
       - 0
       version: "0"