spidr 0.2.3 → 0.2.4

data/ChangeLog.md

@@ -1,3 +1,17 @@
+### 0.2.4 / 2010-05-05
+
+* Added {Spidr::Filters#visit_urls}.
+* Added {Spidr::Filters#visit_urls_like}.
+* Added {Spidr::Filters#ignore_urls}.
+* Added {Spidr::Filters#ignore_urls_like}.
+* Added {Spidr::Page#is_content_type?}.
+* Default {Spidr::Page#body} to an empty String.
+* Default {Spidr::Page#content_type} to an empty String.
+* Default {Spidr::Page#content_types} to an empty Array.
+* Improved reliability of {Spidr::Page#is_redirect?}.
+* Improved content type detection in {Spidr::Page} to handle `Content-Type`
+  headers containing charsets (thanks Josh Lindsey).
+
 ### 0.2.3 / 2010-02-27
 
 * Migrated to Jeweler, for the packaging and releasing RubyGems.

data/README.md

@@ -176,7 +176,7 @@ Skip the processing of links:
 
 ## Requirements
 
-* [nokogiri](http://nokogiri.rubyforge.org/) >= 1.2.0
+* [nokogiri](http://nokogiri.rubyforge.org/) >= 1.3.0
 
 ## Install
 

data/Rakefile

@@ -7,17 +7,19 @@ begin
   Jeweler::Tasks.new do |gem|
     gem.name = 'spidr'
     gem.version = Spidr::VERSION
+    gem.license = 'MIT'
     gem.summary = %Q{A versatile Ruby web spidering library}
     gem.description = %Q{Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.}
     gem.email = 'postmodern.mod3@gmail.com'
     gem.homepage = 'http://github.com/postmodern/spidr'
     gem.authors = ['Postmodern']
-    gem.add_dependency 'nokogiri', '>= 1.2.0'
-    gem.add_development_dependency 'rspec', '>= 1.3.0'
-    gem.add_development_dependency 'yard', '>= 0.5.3'
-    gem.add_development_dependency 'wsoc', '>= 0.1.1'
+    gem.add_dependency 'nokogiri', '>= 1.3.0'
+    gem.add_development_dependency 'rspec', '~> 1.3.0'
+    gem.add_development_dependency 'yard', '~> 0.5.3'
+    gem.add_development_dependency 'wsoc', '~> 0.1.1'
     gem.has_rdoc = 'yard'
   end
+  Jeweler::GemcutterTasks.new
 rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
 end

data/lib/spidr/agent.rb

@@ -702,12 +702,13 @@ module Spidr
     #   Specifies whether the given URL should be visited.
     #
     def visit?(url)
-      (!(visited?(url)) &&
+      !(visited?(url)) &&
        visit_scheme?(url.scheme) &&
        visit_host?(url.host) &&
        visit_port?(url.port) &&
        visit_link?(url.to_s) &&
-       visit_ext?(url.path))
+       visit_url?(url) &&
+       visit_ext?(url.path)
     end
 
     #

data/lib/spidr/filters.rb

@@ -44,6 +44,12 @@ module Spidr
     # @option options [Array<String, Regexp, Proc>] :ignore_links
     #   The patterns which match the links to not visit.
     #
+    # @option options [Array<String, Regexp, Proc>] :urls
+    #   The patterns which match the URLs to visit.
+    #
+    # @option options [Array<String, Regexp, Proc>] :ignore_urls
+    #   The patterns which match the URLs to not visit.
+    #
     # @option options [Array<String, Regexp, Proc>] :exts
     #   The patterns which match the URI path extensions to visit.
     #
@@ -83,6 +89,10 @@ module Spidr
         :accept => options[:links],
         :reject => options[:ignore_links]
       )
+      @url_rules = Rules.new(
+        :accept => options[:urls],
+        :reject => options[:ignore_urls]
+      )
       @ext_rules = Rules.new(
         :accept => options[:exts],
         :reject => options[:ignore_exts]
@@ -125,7 +135,7 @@ module Spidr
     end
 
     #
-    # Adds a given pattern to the visit_hosts.
+    # Adds a given pattern to the {#visit_hosts}.
     #
     # @param [String, Regexp] pattern
     #   The pattern to match host-names with.
@@ -157,7 +167,7 @@ module Spidr
     end
 
     #
-    # Adds a given pattern to the ignore_hosts.
+    # Adds a given pattern to the {#ignore_hosts}.
     #
     # @param [String, Regexp] pattern
     #   The pattern to match host-names with.
@@ -189,7 +199,7 @@ module Spidr
     end
 
     #
-    # Adds a given pattern to the visit_ports.
+    # Adds a given pattern to the {#visit_ports}.
     #
     # @param [Integer, Regexp] pattern
     #   The pattern to match ports with.
@@ -221,7 +231,7 @@ module Spidr
     end
 
     #
-    # Adds a given pattern to the ignore_ports.
+    # Adds a given pattern to the {#ignore_ports}.
     #
     # @param [Integer, Regexp] pattern
     #   The pattern to match ports with.
@@ -248,21 +258,25 @@ module Spidr
     # @return [Array<String, Regexp, Proc>]
     #   The link patterns to visit.
     #
+    # @since 0.2.4
+    #
     def visit_links
       @link_rules.accept
     end
 
     #
-    # Adds a given pattern to the visit_links.
+    # Adds a given pattern to the {#visit_links}
     #
     # @param [String, Regexp] pattern
-    #   The pattern to match links with.
+    #   The pattern to match link with.
     #
     # @yield [link]
     #   If a block is given, it will be used to filter links.
     #
     # @yieldparam [String] link
     #   A link to accept or reject.
+    # 
+    # @since 0.2.4
     #
     def visit_links_like(pattern=nil,&block)
       if pattern
@@ -285,7 +299,7 @@ module Spidr
     end
 
     #
-    # Adds a given pattern to the ignore_links.
+    # Adds a given pattern to the {#ignore_links}.
     #
     # @param [String, Regexp] pattern
     #   The pattern to match links with.
@@ -306,6 +320,78 @@ module Spidr
       return self
     end
 
+    #
+    # Specifies the patterns that match the URLs to visit.
+    #
+    # @return [Array<String, Regexp, Proc>]
+    #   The link patterns to visit.
+    #
+    # @since 0.2.4
+    #
+    def visit_urls
+      @url_rules.accept
+    end
+
+    #
+    # Adds a given pattern to the {#visit_urls}
+    #
+    # @param [String, Regexp] pattern
+    #   The pattern to match URLs with.
+    #
+    # @yield [url]
+    #   If a block is given, it will be used to filter URLs.
+    #
+    # @yieldparam [URI::HTTP, URI::HTTPS] url
+    #   A URL to accept or reject.
+    # 
+    # @since 0.2.4
+    #
+    def visit_urls_like(pattern=nil,&block)
+      if pattern
+        visit_urls << pattern
+      elsif block
+        visit_urls << block
+      end
+
+      return self
+    end
+
+    #
+    # Specifies the patterns that match URLs to not visit.
+    #
+    # @return [Array<String, Regexp, Proc>]
+    #   The URL patterns to not visit.
+    #
+    # @since 0.2.4
+    #
+    def ignore_urls
+      @url_rules.reject
+    end
+
+    #
+    # Adds a given pattern to the {#ignore_urls}.
+    #
+    # @param [String, Regexp] pattern
+    #   The pattern to match URLs with.
+    #
+    # @yield [url]
+    #   If a block is given, it will be used to filter URLs.
+    #
+    # @yieldparam [URI::HTTP, URI::HTTPS] url
+    #   A URL to reject or accept.
+    #
+    # @since 0.2.4
+    #
+    def ignore_urls_like(pattern=nil,&block)
+      if pattern
+        ignore_urls << pattern
+      elsif block
+        ignore_urls << block
+      end
+
+      return self
+    end
+
     #
     # Specifies the patterns that match the URI path extensions to visit.
     #
@@ -317,7 +403,7 @@ module Spidr
     end
 
     #
-    # Adds a given pattern to the visit_exts.
+    # Adds a given pattern to the {#visit_exts}.
     #
     # @param [String, Regexp] pattern
     #   The pattern to match URI path extensions with.
@@ -349,7 +435,7 @@ module Spidr
     end
 
     #
-    # Adds a given pattern to the ignore_exts.
+    # Adds a given pattern to the {#ignore_exts}.
     #
     # @param [String, Regexp] pattern
     #   The pattern to match URI path extensions with.
@@ -428,6 +514,21 @@ module Spidr
       @link_rules.accept?(link)
     end
 
+    #
+    # Determines if a given URL should be visited.
+    #
+    # @param [URI::HTTP, URI::HTTPS] url
+    #   The URL.
+    #
+    # @return [Boolean]
+    #   Specifies whether the given URL should be visited.
+    #
+    # @since 0.2.4
+    #
+    def visit_url?(link)
+      @url_rules.accept?(link)
+    end
+
     #
     # Determines if a given URI path extension should be visited.
     #

data/lib/spidr/page.rb

@@ -61,13 +61,19 @@ module Spidr
     alias ok? is_ok?
 
     #
-    # Determines if the response code is `301` or `307`.
+    # Determines if the response code is `300`, `301`, `302`, `303`
+    # or `307`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is `301` or `307`.
+    #   Specifies whether the response code is a HTTP Redirect code.
     #
     def is_redirect?
-      (code == 301 || code == 307)
+      case code
+      when 300..303, 307
+        true
+      else
+        false
+      end
     end
 
     alias redirect? is_redirect?
@@ -145,7 +151,7 @@ module Spidr
     #   The Content-Type of the page.
     #
     def content_type
-      @response['Content-Type']
+      (@response['Content-Type'] || '')
     end
 
     #
@@ -157,7 +163,7 @@ module Spidr
     # @since 0.2.2
     #
     def content_types
-      @headers['content-type']
+      (@headers['content-type'] || [])
     end
 
     #
@@ -167,7 +173,7 @@ module Spidr
     #   Specifies whether the page is plain-text.
     #
     def plain_text?
-      content_types.include?('text/plain')
+      is_content_type?('text/plain')
     end
 
     alias txt? plain_text?
@@ -179,7 +185,7 @@ module Spidr
     #   Specifies whether the page is HTML document.
     #
     def html?
-      content_types.include?('text/html')
+      is_content_type?('text/html')
     end
 
     #
@@ -189,7 +195,7 @@ module Spidr
     #   Specifies whether the page is XML document.
     #
     def xml?
-      content_types.include?('text/xml')
+      is_content_type?('text/xml')
     end
 
     #
@@ -199,7 +205,7 @@ module Spidr
     #   Specifies whether the page is XML Stylesheet (XSL).
     #
     def xsl?
-      content_types.include?('text/xsl')
+      is_content_type?('text/xsl')
     end
 
     #
@@ -209,8 +215,8 @@ module Spidr
     #   Specifies whether the page is JavaScript.
     #
     def javascript?
-      content_types.include?('text/javascript') || \
-        content_types.include?('application/javascript')
+      is_content_type?('text/javascript') || \
+        is_content_type?('application/javascript')
     end
 
     #
@@ -220,7 +226,7 @@ module Spidr
     #   Specifies whether the page is a CSS stylesheet.
     #
     def css?
-      content_types.include?('text/css')
+      is_content_type?('text/css')
     end
 
     #
@@ -230,8 +236,8 @@ module Spidr
     #   Specifies whether the page is a RSS feed.
     #
     def rss?
-      content_types.include?('application/rss+xml') || \
-        content_types.include?('application/rdf+xml')
+      is_content_type?('application/rss+xml') || \
+        is_content_type?('application/rdf+xml')
     end
 
     #
@@ -241,7 +247,7 @@ module Spidr
     #   Specifies whether the page is an Atom feed.
     #
     def atom?
-      content_types.include?('application/atom+xml')
+      is_content_type?('application/atom+xml')
     end
 
     #
@@ -251,7 +257,7 @@ module Spidr
     #   Specifies whether the page is a MS Word document.
     #
     def ms_word?
-      content_types.include?('application/msword')
+      is_content_type?('application/msword')
     end
 
     #
@@ -261,7 +267,7 @@ module Spidr
     #   Specifies whether the page is a PDF document.
     #
     def pdf?
-      content_types.include?('application/pdf')
+      is_content_type?('application/pdf')
     end
 
     #
@@ -271,7 +277,7 @@ module Spidr
     #   Specifies whether the page is a ZIP archive.
     #
     def zip?
-      content_types.include?('application/zip')
+      is_content_type?('application/zip')
     end
 
     #
@@ -329,7 +335,7 @@ module Spidr
     #   The body of the response.
     #
     def body
-      @response.body
+      (@response.body || '')
     end
 
     #
@@ -344,7 +350,7 @@ module Spidr
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html
     #
     def doc
-      return nil if (body.nil? || body.empty?)
+      return nil if body.empty?
 
       begin
         if html?
@@ -375,10 +381,10 @@ module Spidr
     #
     def search(*paths)
       if doc
-        return doc.search(*paths)
+        doc.search(*paths)
+      else
+        []
       end
-
-      return []
     end
 
     #
@@ -395,10 +401,8 @@ module Spidr
     #
     def at(*arguments)
       if doc
-        return doc.at(*arguments)
+        doc.at(*arguments)
       end
-
-      return nil
     end
 
     alias / search
@@ -412,7 +416,7 @@ module Spidr
     #
     def title
       if (node = at('//title'))
-        return node.inner_text
+        node.inner_text
       end
     end
 
@@ -430,8 +434,7 @@ module Spidr
         urls << url unless (url.nil? || url.empty?)
       }
 
-      case code
-      when 300..303, 307
+      if self.is_redirect?
         location = @headers['location']
 
         if location.kind_of?(Array)
@@ -506,6 +509,22 @@ module Spidr
 
     protected
 
+    #
+    # Determines if any of the content-types of the page include a given
+    # type.
+    #
+    # @param [String] type
+    #   The content-type to test for.
+    #
+    # @return [Boolean]
+    #   Specifies whether the page includes the given content-type.
+    #
+    # @since 0.2.4
+    #
+    def is_content_type?(type)
+      content_types.any? { |content| content.include?(type) }
+    end
+
     #
     # Provides transparent access to the values in `headers`.
     #
@@ -518,6 +537,6 @@ module Spidr
 
       return super(sym,*args,&block)
     end
-
+  
   end
 end

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.2.3'
+  VERSION = '0.2.4'
 end

data/spidr.gemspec

@@ -5,89 +5,90 @@
 
 Gem::Specification.new do |s|
   s.name = %q{spidr}
-  s.version = "0.2.3"
+  s.version = "0.2.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Postmodern"]
-  s.date = %q{2010-02-27}
+  s.date = %q{2010-05-05}
   s.description = %q{Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.}
   s.email = %q{postmodern.mod3@gmail.com}
   s.extra_rdoc_files = [
     "ChangeLog.md",
-    "LICENSE.txt",
-    "README.md"
+     "LICENSE.txt",
+     "README.md"
   ]
   s.files = [
     ".gitignore",
-    ".specopts",
-    ".yardopts",
-    "ChangeLog.md",
-    "LICENSE.txt",
-    "README.md",
-    "Rakefile",
-    "lib/spidr.rb",
-    "lib/spidr/actions.rb",
-    "lib/spidr/actions/actions.rb",
-    "lib/spidr/actions/exceptions.rb",
-    "lib/spidr/actions/exceptions/action.rb",
-    "lib/spidr/actions/exceptions/paused.rb",
-    "lib/spidr/actions/exceptions/skip_link.rb",
-    "lib/spidr/actions/exceptions/skip_page.rb",
-    "lib/spidr/agent.rb",
-    "lib/spidr/auth_credential.rb",
-    "lib/spidr/auth_store.rb",
-    "lib/spidr/cookie_jar.rb",
-    "lib/spidr/events.rb",
-    "lib/spidr/extensions.rb",
-    "lib/spidr/extensions/uri.rb",
-    "lib/spidr/filters.rb",
-    "lib/spidr/page.rb",
-    "lib/spidr/rules.rb",
-    "lib/spidr/sanitizers.rb",
-    "lib/spidr/session_cache.rb",
-    "lib/spidr/spidr.rb",
-    "lib/spidr/version.rb",
-    "spec/actions_spec.rb",
-    "spec/agent_spec.rb",
-    "spec/auth_store_spec.rb",
-    "spec/cookie_jar_spec.rb",
-    "spec/extensions/uri_spec.rb",
-    "spec/filters_spec.rb",
-    "spec/helpers/history.rb",
-    "spec/helpers/page.rb",
-    "spec/helpers/wsoc.rb",
-    "spec/page_examples.rb",
-    "spec/page_spec.rb",
-    "spec/rules_spec.rb",
-    "spec/sanitizers_spec.rb",
-    "spec/session_cache.rb",
-    "spec/spec_helper.rb",
-    "spec/spidr_spec.rb",
-    "spidr.gemspec"
+     ".specopts",
+     ".yardopts",
+     "ChangeLog.md",
+     "LICENSE.txt",
+     "README.md",
+     "Rakefile",
+     "lib/spidr.rb",
+     "lib/spidr/actions.rb",
+     "lib/spidr/actions/actions.rb",
+     "lib/spidr/actions/exceptions.rb",
+     "lib/spidr/actions/exceptions/action.rb",
+     "lib/spidr/actions/exceptions/paused.rb",
+     "lib/spidr/actions/exceptions/skip_link.rb",
+     "lib/spidr/actions/exceptions/skip_page.rb",
+     "lib/spidr/agent.rb",
+     "lib/spidr/auth_credential.rb",
+     "lib/spidr/auth_store.rb",
+     "lib/spidr/cookie_jar.rb",
+     "lib/spidr/events.rb",
+     "lib/spidr/extensions.rb",
+     "lib/spidr/extensions/uri.rb",
+     "lib/spidr/filters.rb",
+     "lib/spidr/page.rb",
+     "lib/spidr/rules.rb",
+     "lib/spidr/sanitizers.rb",
+     "lib/spidr/session_cache.rb",
+     "lib/spidr/spidr.rb",
+     "lib/spidr/version.rb",
+     "spec/actions_spec.rb",
+     "spec/agent_spec.rb",
+     "spec/auth_store_spec.rb",
+     "spec/cookie_jar_spec.rb",
+     "spec/extensions/uri_spec.rb",
+     "spec/filters_spec.rb",
+     "spec/helpers/history.rb",
+     "spec/helpers/page.rb",
+     "spec/helpers/wsoc.rb",
+     "spec/page_examples.rb",
+     "spec/page_spec.rb",
+     "spec/rules_spec.rb",
+     "spec/sanitizers_spec.rb",
+     "spec/session_cache.rb",
+     "spec/spec_helper.rb",
+     "spec/spidr_spec.rb",
+     "spidr.gemspec"
   ]
   s.has_rdoc = %q{yard}
   s.homepage = %q{http://github.com/postmodern/spidr}
+  s.licenses = ["MIT"]
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.6}
   s.summary = %q{A versatile Ruby web spidering library}
   s.test_files = [
-    "spec/agent_spec.rb",
-    "spec/helpers/history.rb",
-    "spec/helpers/wsoc.rb",
-    "spec/helpers/page.rb",
-    "spec/spec_helper.rb",
-    "spec/extensions/uri_spec.rb",
-    "spec/page_spec.rb",
-    "spec/spidr_spec.rb",
-    "spec/sanitizers_spec.rb",
-    "spec/page_examples.rb",
-    "spec/filters_spec.rb",
-    "spec/actions_spec.rb",
-    "spec/rules_spec.rb",
     "spec/auth_store_spec.rb",
-    "spec/cookie_jar_spec.rb",
-    "spec/session_cache.rb"
+     "spec/rules_spec.rb",
+     "spec/session_cache.rb",
+     "spec/spec_helper.rb",
+     "spec/sanitizers_spec.rb",
+     "spec/filters_spec.rb",
+     "spec/page_spec.rb",
+     "spec/spidr_spec.rb",
+     "spec/agent_spec.rb",
+     "spec/cookie_jar_spec.rb",
+     "spec/extensions/uri_spec.rb",
+     "spec/helpers/history.rb",
+     "spec/helpers/page.rb",
+     "spec/helpers/wsoc.rb",
+     "spec/page_examples.rb",
+     "spec/actions_spec.rb"
   ]
 
   if s.respond_to? :specification_version then
@@ -95,21 +96,21 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<nokogiri>, [">= 1.2.0"])
-      s.add_development_dependency(%q<rspec>, [">= 1.3.0"])
-      s.add_development_dependency(%q<yard>, [">= 0.5.3"])
-      s.add_development_dependency(%q<wsoc>, [">= 0.1.1"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.3.0"])
+      s.add_development_dependency(%q<rspec>, ["~> 1.3.0"])
+      s.add_development_dependency(%q<yard>, ["~> 0.5.3"])
+      s.add_development_dependency(%q<wsoc>, ["~> 0.1.1"])
     else
-      s.add_dependency(%q<nokogiri>, [">= 1.2.0"])
-      s.add_dependency(%q<rspec>, [">= 1.3.0"])
-      s.add_dependency(%q<yard>, [">= 0.5.3"])
-      s.add_dependency(%q<wsoc>, [">= 0.1.1"])
+      s.add_dependency(%q<nokogiri>, [">= 1.3.0"])
+      s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+      s.add_dependency(%q<yard>, ["~> 0.5.3"])
+      s.add_dependency(%q<wsoc>, ["~> 0.1.1"])
     end
   else
-    s.add_dependency(%q<nokogiri>, [">= 1.2.0"])
-    s.add_dependency(%q<rspec>, [">= 1.3.0"])
-    s.add_dependency(%q<yard>, [">= 0.5.3"])
-    s.add_dependency(%q<wsoc>, [">= 0.1.1"])
+    s.add_dependency(%q<nokogiri>, [">= 1.3.0"])
+    s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+    s.add_dependency(%q<yard>, ["~> 0.5.3"])
+    s.add_dependency(%q<wsoc>, ["~> 0.1.1"])
   end
 end
 

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - Postmodern
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-27 00:00:00 -08:00
+date: 2010-05-05 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,9 +26,9 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 1
-        - 2
+        - 3
         - 0
-        version: 1.2.0
+        version: 1.3.0
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -36,7 +36,7 @@ dependencies:
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
         - 1
@@ -50,7 +50,7 @@ dependencies:
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
         - 0
@@ -64,7 +64,7 @@ dependencies:
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
         - 0
@@ -132,8 +132,8 @@ files:
 - spidr.gemspec
 has_rdoc: yard
 homepage: http://github.com/postmodern/spidr
-licenses: []
-
+licenses: 
+- MIT
 post_install_message: 
 rdoc_options: 
 - --charset=UTF-8
@@ -161,19 +161,19 @@ signing_key:
 specification_version: 3
 summary: A versatile Ruby web spidering library
 test_files: 
-- spec/agent_spec.rb
-- spec/helpers/history.rb
-- spec/helpers/wsoc.rb
-- spec/helpers/page.rb
+- spec/auth_store_spec.rb
+- spec/rules_spec.rb
+- spec/session_cache.rb
 - spec/spec_helper.rb
-- spec/extensions/uri_spec.rb
+- spec/sanitizers_spec.rb
+- spec/filters_spec.rb
 - spec/page_spec.rb
 - spec/spidr_spec.rb
-- spec/sanitizers_spec.rb
+- spec/agent_spec.rb
+- spec/cookie_jar_spec.rb
+- spec/extensions/uri_spec.rb
+- spec/helpers/history.rb
+- spec/helpers/page.rb
+- spec/helpers/wsoc.rb
 - spec/page_examples.rb
-- spec/filters_spec.rb
 - spec/actions_spec.rb
-- spec/rules_spec.rb
-- spec/auth_store_spec.rb
-- spec/cookie_jar_spec.rb
-- spec/session_cache.rb