spidr 0.3.2 → 0.4.0

data/ChangeLog.md

@@ -1,3 +1,12 @@
+### 0.4.0 / 2011-08-07
+
+* Added {Spidr::Headers#content_charset}.
+* Pass the Page `url` and `content_charset` to Nokogiri in {Spidr::Body#doc}.
+  This ensures that Nokogiri will preserve the body encoding.
+* Made {Spidr::Headers#is_content_type?} public.
+* Allow {Spidr::Headers#is_content_type?} to match the full Content-Type
+  or the sub-type.
+
 ### 0.3.2 / 2011-06-20
 
 * Added separate intitialize methods for {Spidr::Actions}, {Spidr::Events},

data/lib/spidr/body.rb

@@ -27,9 +27,9 @@ module Spidr
       unless body.empty?
         begin
           if html?
-            @doc ||= Nokogiri::HTML(body)
+            @doc ||= Nokogiri::HTML(body, @url.to_s, content_charset)
           elsif (rss? || atom? || xml? || xsl?)
-            @doc ||= Nokogiri::XML(body)
+            @doc ||= Nokogiri::XML(body, @url.to_s, content_charset)
           end
         rescue
         end

data/lib/spidr/filters.rb

@@ -414,7 +414,7 @@ module Spidr
         rescue Gem::LoadError => e
           raise(e)
         rescue ::LoadError
-          STDERR.puts "Warning: cannot load 'net/https', https support disabled"
+          warn "Warning: cannot load 'net/https', https support disabled"
         end
       end
 

data/lib/spidr/headers.rb

@@ -115,6 +115,67 @@ module Spidr
       (headers['content-type'] || [])
     end
 
+    #
+    # The charset included in the Content-Type.
+    #
+    # @return [String, nil]
+    #   The charset of the content.
+    #
+    # @since 0.4.0
+    #
+    def content_charset
+      content_types.each do |value|
+        if value.include?(';')
+          value.split(';').each do |param|
+            param.strip!
+
+            if param.start_with?('charset=')
+              return param.split('=',2).last
+            end
+          end
+        end
+      end
+
+      return nil
+    end
+
+    #
+    # Determines if any of the content-types of the page include a given
+    # type.
+    #
+    # @param [String] type
+    #   The content-type to test for.
+    #
+    # @return [Boolean]
+    #   Specifies whether the page includes the given content-type.
+    #
+    # @example Match the Content-Type
+    #   page.is_content_type?('application/json')
+    #
+    # @example Match the sub-type of the Content-Type
+    #   page.is_content_type?('json')
+    #
+    # @since 0.4.0
+    #
+    def is_content_type?(type)
+      if type.include?('/')
+        # otherwise only match the first param
+        content_types.any? do |value|
+          value = value.split(';',2).first
+
+          value == type
+        end
+      else
+        # otherwise only match the sub-type
+        content_types.any? do |value|
+          value = value.split(';',2).first
+          value = value.split('/',2).last
+
+          value == type
+        end
+      end
+    end
+
     #
     # Determines if the page is plain-text.
     #
@@ -291,35 +352,19 @@ module Spidr
     def cookie_params
       params = {}
 
-      cookies.each do |cookie|
-        cookie.split('; ').each do |key_value|
-          key, value = key_value.split('=',2)
+      cookies.each do |value|
+        value.split(';').each do |param|
+          param.strip!
+
+          name, value = param.split('=',2)
 
-          unless RESERVED_COOKIE_NAMES.include?(key)
-            params[key] = (value || '')
+          unless RESERVED_COOKIE_NAMES.include?(name)
+            params[name] = (value || '')
           end
         end
       end
 
       return params
     end
-
-    protected
-
-    #
-    # Determines if any of the content-types of the page include a given
-    # type.
-    #
-    # @param [String] type
-    #   The content-type to test for.
-    #
-    # @return [Boolean]
-    #   Specifies whether the page includes the given content-type.
-    #
-    # @since 0.2.4
-    #
-    def is_content_type?(type)
-      content_types.any? { |content| content.include?(type) }
-    end
   end
 end

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.3.2'
+  VERSION = '0.4.0'
 end

data/spec/page_spec.rb

@@ -17,7 +17,7 @@ describe Page do
     end
 
     it "should have a content-type" do
-      @page.content_type.should =~ /text\/html/
+      @page.content_type.should include('text/html')
     end
 
     it "should be a html page" do
@@ -54,7 +54,7 @@ describe Page do
     end
 
     it "should have a content-type" do
-      @page.content_type.should =~ /text\/plain/
+      @page.content_type.should include('text/plain')
     end
 
     it "should be a txt page" do

data/spidr.gemspec

@@ -3,6 +3,8 @@
 require 'yaml'
 
 Gem::Specification.new do |gemspec|
+  root = File.dirname(__FILE__)
+  lib_dir = File.join(root,'lib')
   files = if File.directory?('.git')
             `git ls-files`.split($/)
           elsif File.directory?('.hg')
@@ -23,12 +25,12 @@ Gem::Specification.new do |gemspec|
   }
 
   version = {
-    :file => 'lib/spidr/version.rb',
+    :file => 'spidr/version',
     :constant => 'Spidr::VERSION'
   }
 
   defaults = {
-    'name' => File.basename(File.dirname(__FILE__)),
+    'name' => File.basename(root),
     'files' => files,
     'executables' => filter_files['bin/*'].map { |path| File.basename(path) },
     'test_files' => filter_files['{test/{**/}*_test.rb,spec/{**/}*_spec.rb}'],
@@ -40,8 +42,10 @@ Gem::Specification.new do |gemspec|
   gemspec.name = metadata.fetch('name',defaults[:name])
   gemspec.version = if metadata['version']
                       metadata['version']
-                    elsif File.file?(version[:file])
-                      require File.join('.',version[:file])
+                    else
+                      $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
+
+                      require version[:file]
                       eval(version[:constant])
                     end
 

metadata

@@ -1,60 +1,60 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: spidr
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 0.4.0
   prerelease: 
-  version: 0.3.2
 platform: ruby
-authors: 
+authors:
 - Postmodern
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-06-20 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-08-07 00:00:00.000000000 -07:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &15697980 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: "1.3"
+      - !ruby/object:Gem::Version
+        version: '1.3'
   type: :runtime
   prerelease: false
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
+  version_requirements: *15697980
+- !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &15693140 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
+  version_requirements: *15693140
+- !ruby/object:Gem::Dependency
   name: yard
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &15668040 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *id003
-description: Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.
-email: 
+  version_requirements: *15668040
+description: Spidr is a versatile Ruby web spidering library that can spider a site,
+  multiple domains, certain links or infinitely. Spidr is designed to be fast and
+  easy to use.
+email:
 - postmodern.mod3@gmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.md
-files: 
+files:
 - .rspec
 - .yardopts
 - ChangeLog.md
@@ -105,34 +105,33 @@ files:
 - spec/spec_helper.rb
 - spec/spidr_spec.rb
 - spidr.gemspec
+has_rdoc: yard
 homepage: http://github.com/postmodern/spidr
-licenses: 
+licenses:
 - MIT
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
-
 rubyforge_project: spidr
-rubygems_version: 1.8.5
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: A versatile Ruby web spidering library
-test_files: 
+test_files:
 - spec/agent_spec.rb
 - spec/actions_spec.rb
 - spec/rules_spec.rb