webrobots 0.0.6 → 0.0.7

data/VERSION

@@ -1 +1 @@
-0.0.6
+0.0.7

data/lib/webrobots/robotstxt.rb

@@ -8,7 +8,6 @@ require 'racc/parser.rb'
 
 
 require 'strscan'
-require 'uri'
 
 class WebRobots
   class Error < StandardError
@@ -20,7 +19,7 @@ class WebRobots
   class RobotsTxt
 class Parser < Racc::Parser
 
-module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 164)
+module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 163)
 
       def initialize(target = nil)
         super()
@@ -34,10 +33,10 @@ module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 164)
       end
 
       KNOWN_TOKENS = %w[User-agent Allow Disallow Crawl-delay Sitemap]
-      RE_KNOWN_TOKENS = /#{KNOWN_TOKENS.join('|')}/i
+      RE_KNOWN_TOKENS = /\A(#{KNOWN_TOKENS.map { |t| Regexp.quote(t) }.join('|')})\z/i
 
       def parse(input, site)
-        @q = []
+        @q ||= []
         @errors = []
         @lineno = 1
         @site = site
@@ -71,14 +70,15 @@ module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 164)
                 parse_error @lineno, "unexpected characters: %s" % s.check(/.*/)
               end
               value_expected = false
-            else
-              if t = s.scan(RE_KNOWN_TOKENS)
+            elsif t = s.scan(/[^\x00-\x1f\x7f()<>@,;:\\"\/\[\]?={}]+/)
+              case t
+              when RE_KNOWN_TOKENS
                 @q << [t.downcase, t]
-              elsif t = s.scan(/[^\x00-\x1f\x7f()<>@,;:\\"\/\[\]?={}]+/)
-                @q << [:TOKEN, t]
               else
-                parse_error "unexpected characters: %s" % s.check(/.*/)
+                @q << [:TOKEN, t]
               end
+            else
+              parse_error "unexpected characters: %s" % s.check(/.*/)
             end
           end
         end
@@ -90,6 +90,8 @@ module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 164)
         do_parse
       rescue Racc::ParseError => e
         raise ParseError, e.message
+      ensure
+        @q.clear
       end
 
       def next_token
@@ -283,7 +285,7 @@ Racc_token_to_s_table = [
   "opt_blanklines",
   "body",
   "@1",
-  "blocks",
+  "records",
   "blanklines",
   "blankline",
   "eol",
@@ -614,12 +616,8 @@ Disallow: /
             @options[ruleline.token.downcase] = ruleline.value
           end
         }
-        @acls.sort! { |a, b|
-          [
-            b.value.length, b.is_a?(AllowLine) ? 1 : 0
-          ] <=> [
-            a.value.length, a.is_a?(AllowLine) ? 1 : 0
-          ]
+        @acls.replace @acls.sort_by { |x|
+          [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
         }
       end
 

data/lib/webrobots/robotstxt.ry

@@ -15,7 +15,7 @@ rule
 			  }
 
   body			:
-			| blocks
+			| records
 			  opt_blanklines
 
   opt_blanklines	:
@@ -48,7 +48,7 @@ rule
 			    @sitemaps << val[3]
 			  }
 
-  blocks		: record
+  records		: record
 			  {
 			    result = []
 			    result << val[0]
@@ -57,13 +57,13 @@ rule
 			  {
 			    result = []
 			  }
-			| blocks
+			| records
 			  blanklines
 			  record
 			  {
 			    result << val[2]
 			  }
-			| blocks
+			| records
 			  blanklines
 			  rulelines
 			  {
@@ -72,7 +72,7 @@ rule
 			        [@site.to_s, @rulelinenos[i], line.token] if $VERBOSE
 			    }
 			  }
-			| blocks
+			| records
 			  blanklines
 			  commentblock
 
@@ -150,7 +150,6 @@ rule
 ---- header
 
 require 'strscan'
-require 'uri'
 
 class WebRobots
   class Error < StandardError
@@ -174,10 +173,10 @@ class WebRobots
       end
 
       KNOWN_TOKENS = %w[User-agent Allow Disallow Crawl-delay Sitemap]
-      RE_KNOWN_TOKENS = /#{KNOWN_TOKENS.join('|')}/i
+      RE_KNOWN_TOKENS = /\A(#{KNOWN_TOKENS.map { |t| Regexp.quote(t) }.join('|')})\z/i
 
       def parse(input, site)
-        @q = []
+        @q ||= []
         @errors = []
         @lineno = 1
         @site = site
@@ -211,14 +210,15 @@ class WebRobots
                 parse_error @lineno, "unexpected characters: %s" % s.check(/.*/)
               end
               value_expected = false
-            else
-              if t = s.scan(RE_KNOWN_TOKENS)
+            elsif t = s.scan(/[^\x00-\x1f\x7f()<>@,;:\\"\/\[\]?={}]+/)
+              case t
+              when RE_KNOWN_TOKENS
                 @q << [t.downcase, t]
-              elsif t = s.scan(/[^\x00-\x1f\x7f()<>@,;:\\"\/\[\]?={}]+/)
-                @q << [:TOKEN, t]
               else
-                parse_error "unexpected characters: %s" % s.check(/.*/)
+                @q << [:TOKEN, t]
               end
+            else
+              parse_error "unexpected characters: %s" % s.check(/.*/)
             end
           end
         end
@@ -230,6 +230,8 @@ class WebRobots
         do_parse
       rescue Racc::ParseError => e
         raise ParseError, e.message
+      ensure
+        @q.clear
       end
 
       def next_token
@@ -344,12 +346,8 @@ Disallow: /
             @options[ruleline.token.downcase] = ruleline.value
           end
         }
-        @acls.sort! { |a, b|
-          [
-            b.value.length, b.is_a?(AllowLine) ? 1 : 0
-          ] <=> [
-            a.value.length, a.is_a?(AllowLine) ? 1 : 0
-          ]
+        @acls.replace @acls.sort_by { |x|
+          [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
         }
       end
 

data/lib/webrobots.rb

@@ -1,6 +1,7 @@
 require 'webrobots/robotstxt'
 require 'uri'
 require 'net/https'
+require 'thread'
 if defined?(Nokogiri)
   require 'webrobots/nokogiri'
 else
@@ -19,6 +20,7 @@ class WebRobots
   def initialize(user_agent, options = nil)
     @user_agent = user_agent
     @parser = RobotsTxt::Parser.new(user_agent)
+    @parser_mutex = Mutex.new
 
     options ||= {}
     @http_get = options[:http_get] || method(:http_get)
@@ -133,7 +135,9 @@ class WebRobots
     rescue => e
       return RobotsTxt.unfetchable(site, e, @user_agent)
     end
-    @parser.parse!(body, site)
+    @parser_mutex.synchronize {
+      @parser.parse!(body, site)
+    }
   end
 
   def http_get(uri)

data/test/test_webrobots.rb

@@ -89,6 +89,7 @@ class TestWebRobots < Test::Unit::TestCase
 # Punish evil bots
 User-Agent: evil
 Disallow: /
+Disallow-Not: /	# parser teaser
 
 User-Agent: good
 # Be generous to good bots
@@ -172,7 +173,9 @@ Disallow: /~joe/index.html
     end
 
     should "properly restrict access" do
-      assert  @robots_good.allowed?('http://www.example.org/index.html')
+      assert_nothing_raised {
+        assert  @robots_good.allowed?('http://www.example.org/index.html')
+      }
       assert !@robots_good.allowed?('http://www.example.org/2heavy/index.php')
       assert  @robots_good.allowed?('http://www.example.org/2HEAVY/index.php')
       assert !@robots_good.allowed?(URI('http://www.example.org/2heavy/index.php'))

data/webrobots.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{webrobots}
-  s.version = "0.0.6"
+  s.version = "0.0.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Akinori MUSHA"]
-  s.date = %q{2011-01-09}
+  s.date = %q{2011-02-01}
   s.description = %q{This library helps write robots.txt compliant web robots in Ruby.
 }
   s.email = %q{knu@idaemons.org}
@@ -35,7 +35,7 @@ Gem::Specification.new do |s|
   ]
   s.licenses = ["2-clause BSDL"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.4.1}
+  s.rubygems_version = %q{1.4.2}
   s.summary = %q{A Ruby library to help write robots.txt compliant web robots}
   s.test_files = [
     "test/helper.rb",

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: webrobots
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: 
   segments: 
   - 0
   - 0
-  - 6
-  version: 0.0.6
+  - 7
+  version: 0.0.7
 platform: ruby
 authors: 
 - Akinori MUSHA
@@ -15,10 +15,12 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-09 00:00:00 +09:00
+date: 2011-02-01 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: racc
   type: :runtime
   version_requirements: &id001 !ruby/object:Gem::Requirement 
     none: false
@@ -30,9 +32,9 @@ dependencies:
         - 0
         version: "0"
   requirement: *id001
-  prerelease: false
-  name: racc
 - !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: nokogiri
   type: :runtime
   version_requirements: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -46,9 +48,9 @@ dependencies:
         - 4
         version: 1.4.4
   requirement: *id002
-  prerelease: false
-  name: nokogiri
 - !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: shoulda
   type: :development
   version_requirements: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -60,9 +62,9 @@ dependencies:
         - 0
         version: "0"
   requirement: *id003
-  prerelease: false
-  name: shoulda
 - !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: bundler
   type: :development
   version_requirements: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -76,9 +78,9 @@ dependencies:
         - 0
         version: 1.0.0
   requirement: *id004
-  prerelease: false
-  name: bundler
 - !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: jeweler
   type: :development
   version_requirements: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -92,9 +94,9 @@ dependencies:
         - 1
         version: 1.5.1
   requirement: *id005
-  prerelease: false
-  name: jeweler
 - !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: rcov
   type: :development
   version_requirements: &id006 !ruby/object:Gem::Requirement 
     none: false
@@ -106,9 +108,9 @@ dependencies:
         - 0
         version: "0"
   requirement: *id006
-  prerelease: false
-  name: rcov
 - !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: racc
   type: :development
   version_requirements: &id007 !ruby/object:Gem::Requirement 
     none: false
@@ -120,8 +122,6 @@ dependencies:
         - 0
         version: "0"
   requirement: *id007
-  prerelease: false
-  name: racc
 description: |
   This library helps write robots.txt compliant web robots in Ruby.
 
@@ -178,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.4.1
+rubygems_version: 1.4.2
 signing_key: 
 specification_version: 3
 summary: A Ruby library to help write robots.txt compliant web robots