webrobots 0.0.9 → 0.0.10

data/Gemfile

@@ -10,6 +10,6 @@ group :development do
   gem "racc", ">= 0"
   gem "shoulda", ">= 0"
   gem "bundler", "~> 1.0.0"
-  gem "jeweler", "~> 1.5.1"
+  gem "jeweler", "~> 1.6.2"
   gem "rcov", ">= 0"
 end

data/Gemfile.lock

@@ -2,13 +2,13 @@ GEM
   remote: http://rubygems.org/
   specs:
     git (1.2.5)
-    jeweler (1.5.2)
-      bundler (~> 1.0.0)
+    jeweler (1.6.2)
+      bundler (~> 1.0)
       git (>= 1.2.5)
       rake
-    nokogiri (1.4.4)
+    nokogiri (1.4.6)
     racc (1.4.6)
-    rake (0.8.7)
+    rake (0.9.2)
     rcov (0.9.9)
     shoulda (2.11.3)
 
@@ -17,7 +17,7 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.0.0)
-  jeweler (~> 1.5.1)
+  jeweler (~> 1.6.2)
   nokogiri (>= 1.4.4)
   racc
   rcov

data/Rakefile

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'rubygems'
 require 'bundler'
 begin
@@ -21,9 +23,7 @@ This library helps write robots.txt compliant web robots in Ruby.
   EOS
   gem.email = "knu@idaemons.org"
   gem.authors = ["Akinori MUSHA"]
-  # Include your dependencies below. Runtime dependencies are required when using your gem,
-  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
-  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new
 
@@ -39,6 +39,7 @@ Rcov::RcovTask.new do |test|
   test.libs << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
 end
 
 task :default => :test

data/VERSION

@@ -1 +1 @@
-0.0.9
+0.0.10

data/lib/webrobots/robotstxt.rb

@@ -46,7 +46,7 @@ module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 163)
         value_expected = false
 
         until s.eos?
-          if t = s.scan(/[ \t]*\r?\n/)
+          if t = s.scan(/[ \t]*(?:\r?\n|\z)/)
             if value_expected
               @q << [:VALUE, '']
             end

data/lib/webrobots/robotstxt.ry

@@ -186,7 +186,7 @@ class WebRobots
         value_expected = false
 
         until s.eos?
-          if t = s.scan(/[ \t]*\r?\n/)
+          if t = s.scan(/[ \t]*(?:\r?\n|\z)/)
             if value_expected
               @q << [:VALUE, '']
             end

data/test/test_webrobots.rb

@@ -545,4 +545,41 @@ Disallow: /
       assert_equal 'content', body
     end
   end
+
+  context "robots.txt with a space at the end of the last line" do
+    setup do
+      @robots = WebRobots.new('RandomBot', :http_get => lambda { |uri|
+        res = case uri.to_s
+          when 'http://site1.example.com/robots.txt'
+          <<-'TXT'
+User-agent: *
+Request-rate: 1/30
+Disallow: /util/
+
+Sitemap: http://site1.example.com/text/sitemap.xml
+ 
+TXT
+        when 'http://site2.example.com/robots.txt'
+          <<-'TXT'
+User-agent: *
+Request-rate: 1/30
+Disallow: /util/
+
+Sitemap: http://site2.example.com/text/sitemap.xml 
+TXT
+        else
+          raise "#{uri} is not supposed to be fetched"
+        end
+        # This chomp is actually key to the test.  Remove the final EOL.
+        # The final line should be the one ending with the space.
+        res.chomp
+      })
+    end
+
+    should "be properly parsed" do
+      assert_equal(["http://site1.example.com/text/sitemap.xml"], @robots.sitemaps("http://site1.example.com/"))
+      assert_equal(["http://site2.example.com/text/sitemap.xml"], @robots.sitemaps("http://site2.example.com/"))
+    end
+  end
+
 end

data/webrobots.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{webrobots}
-  s.version = "0.0.9"
+  s.version = "0.0.10"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Akinori MUSHA"]
-  s.date = %q{2011-04-11}
+  s.authors = [%q{Akinori MUSHA}]
+  s.date = %q{2011-07-01}
   s.description = %q{This library helps write robots.txt compliant web robots in Ruby.
 }
   s.email = %q{knu@idaemons.org}
@@ -33,14 +33,10 @@ Gem::Specification.new do |s|
     "test/test_webrobots.rb",
     "webrobots.gemspec"
   ]
-  s.licenses = ["2-clause BSDL"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.6.2}
+  s.licenses = [%q{2-clause BSDL}]
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.5}
   s.summary = %q{A Ruby library to help write robots.txt compliant web robots}
-  s.test_files = [
-    "test/helper.rb",
-    "test/test_webrobots.rb"
-  ]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
@@ -50,14 +46,14 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<racc>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.5.1"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.2"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
     else
       s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_dependency(%q<racc>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.2"])
       s.add_dependency(%q<rcov>, [">= 0"])
     end
   else
@@ -65,7 +61,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<racc>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.2"])
     s.add_dependency(%q<rcov>, [">= 0"])
   end
 end

metadata

@@ -1,125 +1,92 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: webrobots
-version: !ruby/object:Gem::Version 
-  hash: 13
+version: !ruby/object:Gem::Version
+  version: 0.0.10
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 9
-  version: 0.0.9
 platform: ruby
-authors: 
+authors:
 - Akinori MUSHA
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-04-11 00:00:00 +09:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-07-01 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: nokogiri
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
+  requirement: &17196464340 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 1
-        - 4
-        - 4
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.4.4
-  prerelease: false
   type: :runtime
-  requirement: *id001
-- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: *17196464340
+- !ruby/object:Gem::Dependency
   name: racc
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
+  requirement: &17196463180 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  prerelease: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id002
-- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: *17196463180
+- !ruby/object:Gem::Dependency
   name: shoulda
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
+  requirement: &17196462340 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  prerelease: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id003
-- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: *17196462340
+- !ruby/object:Gem::Dependency
   name: bundler
-  version_requirements: &id004 !ruby/object:Gem::Requirement 
+  requirement: &17196461320 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
+      - !ruby/object:Gem::Version
         version: 1.0.0
-  prerelease: false
   type: :development
-  requirement: *id004
-- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: *17196461320
+- !ruby/object:Gem::Dependency
   name: jeweler
-  version_requirements: &id005 !ruby/object:Gem::Requirement 
+  requirement: &17196460320 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 1
-        - 5
-        - 1
-        version: 1.5.1
-  prerelease: false
+      - !ruby/object:Gem::Version
+        version: 1.6.2
   type: :development
-  requirement: *id005
-- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: *17196460320
+- !ruby/object:Gem::Dependency
   name: rcov
-  version_requirements: &id006 !ruby/object:Gem::Requirement 
+  requirement: &17196459400 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  prerelease: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id006
-description: |
-  This library helps write robots.txt compliant web robots in Ruby.
+  prerelease: false
+  version_requirements: *17196459400
+description: ! 'This library helps write robots.txt compliant web robots in Ruby.
 
+'
 email: knu@idaemons.org
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.txt
 - README.rdoc
-files: 
+files:
 - .document
 - Gemfile
 - Gemfile.lock
@@ -134,40 +101,32 @@ files:
 - test/helper.rb
 - test/test_webrobots.rb
 - webrobots.gemspec
-has_rdoc: true
 homepage: 
-licenses: 
+licenses:
 - 2-clause BSDL
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: 1141598142888730213
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: A Ruby library to help write robots.txt compliant web robots
-test_files: 
-- test/helper.rb
-- test/test_webrobots.rb
+test_files: []