webrobots 0.0.12 → 0.0.13

data/Gemfile

@@ -2,14 +2,16 @@ source "http://rubygems.org"
 # Add dependencies required to use your gem here.
 # Example:
 #   gem "activesupport", ">= 2.3.5"
-gem "nokogiri", ">= 1.4.4"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "racc", ">= 0"
   gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.0.0"
+  gem "bundler", ">= 1.0.0"
   gem "jeweler", "~> 1.6.4"
-  gem "rcov", ">= 0"
+  gem "rcov", "~> 0.9.11"
+
+  # To test the webrobots/nokogiri module.
+  gem "nokogiri", ">= 1.4.4"
 end

data/Gemfile.lock

@@ -7,18 +7,18 @@ GEM
       git (>= 1.2.5)
       rake
     nokogiri (1.5.0)
-    racc (1.4.6)
-    rake (0.9.2)
-    rcov (0.9.10)
+    racc (1.4.7)
+    rake (0.9.2.2)
+    rcov (0.9.11)
     shoulda (2.11.3)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.0.0)
+  bundler (>= 1.0.0)
   jeweler (~> 1.6.4)
   nokogiri (>= 1.4.4)
   racc
-  rcov
+  rcov (~> 0.9.11)
   shoulda

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2010, 2011 Akinori MUSHA
+Copyright (c) 2010, 2011, 2012 Akinori MUSHA
 
 All rights reserved.
 

data/README.rdoc

@@ -21,7 +21,6 @@ This is a library to help write robots.txt compliant web robots.
 == Requirements
 
 - Ruby 1.8.7 or 1.9.2+
-- Nokogiri 1.4.4+
 
 == Contributing to webrobots
 
@@ -35,6 +34,5 @@ This is a library to help write robots.txt compliant web robots.
 
 == Copyright
 
-Copyright (c) 2010, 2011 Akinori MUSHA. See LICENSE.txt for
+Copyright (c) 2010, 2011, 2012 Akinori MUSHA. See LICENSE.txt for
 further details.
-

data/VERSION

@@ -1 +1 @@
-0.0.12
+0.0.13

data/lib/webrobots.rb

@@ -145,8 +145,12 @@ class WebRobots
     referer = nil
     10.times {
       http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = uri.is_a?(URI::HTTPS)
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      if http.use_ssl = uri.is_a?(URI::HTTPS)
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        http.cert_store = OpenSSL::X509::Store.new.tap { |store|
+          store.set_default_paths
+        }
+      end
       header = { 'User-Agent' => @user_agent }
       header['Referer'] = referer if referer
       # header is destroyed by this in ruby 1.9.2!

data/lib/webrobots/robotstxt.rb

@@ -1,6 +1,6 @@
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by Racc 1.4.6
+# This file is automatically generated by Racc 1.4.7
 # from Racc grammer file "".
 #
 
@@ -14,12 +14,23 @@ class WebRobots
   end
 
   class ParseError < Error
+    # The site's root URI
+    attr_reader :site
+
+    def initialize(message, site)
+      @message = message
+      @site = site
+    end
+
+    def to_s
+      @message
+    end
   end
 
   class RobotsTxt
 class Parser < Racc::Parser
 
-module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 158)
+module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 169)
 
       def initialize(target = nil)
         super()
@@ -90,7 +101,7 @@ module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 158)
 
         do_parse
       rescue Racc::ParseError => e
-        raise ParseError, e.message
+        raise ParseError.new(e.message, @site)
       ensure
         @q.clear
       end
@@ -606,7 +617,7 @@ Disallow: /
           else
             @options[ruleline.token.downcase] = ruleline.value
           end
-        }
+        } if rulelines
         @acls.replace @acls.sort_by { |x|
           [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
         }

data/lib/webrobots/robotstxt.ry

@@ -151,6 +151,17 @@ class WebRobots
   end
 
   class ParseError < Error
+    # The site's root URI
+    attr_reader :site
+
+    def initialize(message, site)
+      @message = message
+      @site = site
+    end
+
+    def to_s
+      @message
+    end
   end
 
   class RobotsTxt
@@ -225,7 +236,7 @@ class WebRobots
 
         do_parse
       rescue Racc::ParseError => e
-        raise ParseError, e.message
+        raise ParseError.new(e.message, @site)
       ensure
         @q.clear
       end
@@ -341,7 +352,7 @@ Disallow: /
           else
             @options[ruleline.token.downcase] = ruleline.value
           end
-        }
+        } if rulelines
         @acls.replace @acls.sort_by { |x|
           [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
         }

data/test/test_webrobots.rb

@@ -327,7 +327,9 @@ Allow: /2heavy/*.html
       robots.reset(url)
 
       assert robots.allowed?(url)
-      assert_instance_of WebRobots::ParseError, robots.error(url)
+      error = robots.error(url)
+      assert_instance_of WebRobots::ParseError, error
+      assert_equal URI('http://www.example.org/'), error.site
       assert_raise(WebRobots::ParseError) {
         robots.error!(url)
       }
@@ -472,6 +474,7 @@ Disallow: /
     should "be parsed for major sites" do
       assert_nothing_raised {
         assert !@testbot.allowed?("http://www.google.com/search")
+        assert !@testbot.allowed?("https://www.google.com/search")
         assert !@testbot.allowed?("http://www.google.com/news/section?pz=1&cf=all&ned=jp&topic=y&ict=ln")
         assert @testbot.allowed?("http://www.google.com/news/directory?pz=1&cf=all&ned=us&hl=en&sort=users&category=6")
       }
@@ -655,4 +658,25 @@ TXT
       assert !@fetched
     end
   end
+
+  context "robots.txt with just user-agent & sitemap and no blank line between them" do
+    setup do
+      @robots = WebRobots.new('RandomBot', :http_get => lambda { |uri|
+        res = case uri.to_s
+          when 'http://site1.example.com/robots.txt'
+          <<-'TXT'
+User-agent: *
+Sitemap: http://site1.example.com/text/sitemap.xml
+TXT
+        else
+          raise "#{uri} is not supposed to be fetched"
+        end
+      })
+    end
+
+    should "be properly parsed" do
+      assert @robots.allowed?("http://site1.example.com/foo")
+      assert_equal(["http://site1.example.com/text/sitemap.xml"], @robots.sitemaps("http://site1.example.com/"))
+    end
+  end
 end

data/webrobots.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "webrobots"
-  s.version = "0.0.12"
+  s.version = "0.0.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Akinori MUSHA"]
-  s.date = "2011-09-26"
+  s.date = "2012-01-24"
   s.description = "This library helps write robots.txt compliant web robots in Ruby.\n"
   s.email = "knu@idaemons.org"
   s.extra_rdoc_files = [
@@ -35,34 +35,34 @@ Gem::Specification.new do |s|
   s.homepage = "https://github.com/knu/webrobots"
   s.licenses = ["2-clause BSDL"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
+  s.rubygems_version = "1.8.15"
   s.summary = "A Ruby library to help write robots.txt compliant web robots"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_development_dependency(%q<racc>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<bundler>, [">= 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<rcov>, ["~> 0.9.11"])
+      s.add_development_dependency(%q<nokogiri>, [">= 1.4.4"])
     else
-      s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_dependency(%q<racc>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<bundler>, [">= 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<rcov>, ["~> 0.9.11"])
+      s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
     end
   else
-    s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
     s.add_dependency(%q<racc>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<bundler>, [">= 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<rcov>, ["~> 0.9.11"])
+    s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
   end
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: webrobots
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: 
   segments: 
   - 0
   - 0
-  - 12
-  version: 0.0.12
+  - 13
+  version: 0.0.13
 platform: ruby
 authors: 
 - Akinori MUSHA
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-26 00:00:00 Z
+date: 2012-01-24 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   requirement: &id001 !ruby/object:Gem::Requirement 
@@ -23,16 +23,14 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 15
+        hash: 3
         segments: 
-        - 1
-        - 4
-        - 4
-        version: 1.4.4
+        - 0
+        version: "0"
   version_requirements: *id001
-  name: nokogiri
+  name: racc
   prerelease: false
-  type: :runtime
+  type: :development
 - !ruby/object:Gem::Dependency 
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -44,7 +42,7 @@ dependencies:
         - 0
         version: "0"
   version_requirements: *id002
-  name: racc
+  name: shoulda
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency 
@@ -53,12 +51,14 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 23
         segments: 
+        - 1
         - 0
-        version: "0"
+        - 0
+        version: 1.0.0
   version_requirements: *id003
-  name: shoulda
+  name: bundler
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency 
@@ -67,14 +67,14 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 23
+        hash: 7
         segments: 
         - 1
-        - 0
-        - 0
-        version: 1.0.0
+        - 6
+        - 4
+        version: 1.6.4
   version_requirements: *id004
-  name: bundler
+  name: jeweler
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency 
@@ -83,14 +83,14 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
+        hash: 45
         segments: 
-        - 1
-        - 6
-        - 4
-        version: 1.6.4
+        - 0
+        - 9
+        - 11
+        version: 0.9.11
   version_requirements: *id005
-  name: jeweler
+  name: rcov
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency 
@@ -99,12 +99,14 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 15
         segments: 
-        - 0
-        version: "0"
+        - 1
+        - 4
+        - 4
+        version: 1.4.4
   version_requirements: *id006
-  name: rcov
+  name: nokogiri
   prerelease: false
   type: :development
 description: |
@@ -162,7 +164,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: A Ruby library to help write robots.txt compliant web robots