robotx 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 161a4310d0e1b28e499ce5dd6226125c6e345dd6
-  data.tar.gz: 2c33050af6edcdc516611e7eb8e1efc5a497ecf5
+  metadata.gz: 38247d7fb4545eed6e9a78ba05a9499fec08238c
+  data.tar.gz: 8042484caaa6979bd9a4eb6e7c704d493aac06d1
 SHA512:
-  metadata.gz: 6dc47d5c31e4629bb462ed353e31ec5e2b5b98fbf2a56363d87c8e9c9a8ed5a611341d88268f29b66b5b268acf4dce8e7766b7be0d6f189f1696544602d86d89
-  data.tar.gz: b939d2cf78e12054a92f8693ad35e5cc55efe75098df8b2e0c76c4347e3ad3763a15812a51968eba40c72b849c51b5373abddc9f0dfd92d49c3c2b1b85d59e84
+  metadata.gz: f0e97890f42ccaa6e24aa194f34088252df7861290cca8bd2d8acc4266e57f12fe419c660f3491895d02530aee36b6f57d42177f0befab6761b960f6bac2dd71
+  data.tar.gz: 9c73d05ad22f458c05bed28062aa65914c45b0c9dc90400e66963e621c53de22d135c1393a483f7bb9152bc157187ffba7307414907a4cb2d5b67f8f89a9d5ae

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    robotx (0.1.0)
+    robotx (0.1.1)
 
 GEM
   remote: https://rubygems.org/

data/lib/robotx.rb

@@ -77,24 +77,24 @@ private
           hash[agent] ||= {}
         when 'allow'
           hash[agent]['allow'] ||= []
-          hash[agent]['allow'] << value.sub(/(\/){2,}$/, '')
+          hash[agent]['allow'] << strip_slashes(value)
         when 'disallow'
-          # Disallow: '' means Allow: '/'
+          # Disallow: '' equals Allow: '/'
           if value.empty?
             hash[agent]['allow'] ||= []
             hash[agent]['allow'] << '/'
           else
             hash[agent]['disallow'] ||= []
-            hash[agent]['disallow'] << value.sub(/(\/){2,}$/, '')
+            hash[agent]['disallow'] << strip_slashes(value)
           end
         when 'crawl-delay'
           hash[agent]['crawl-delay'] = value.to_i
         when 'sitemap'
           hash['sitemap'] ||= []
-          hash['sitemap'] << value.sub(/(\/){2,}$/, '')
+          hash['sitemap'] << strip_slashes(value).sub(/\/*$/, '')
         else
           hash[key] ||= []
-          hash[key] << value.sub(/(\/){2,}$/, '')
+          hash[key] << strip_slashes(value)
         end
       end
     end
@@ -102,13 +102,30 @@ private
     {}
   end
 
+  def strip_slashes(value='')
+    return value.sub(/\/*$/, '/')
+  end
+
+  def regex_value(value='')
+    return strip_slashes(value).gsub(/\*/,'.*').gsub(/\?/,'\?')
+  end
+
+  def disallow_regex
+    disallow_data = @robots_data.fetch(@user_agent, {}).fetch('disallow', [])
+    @disallow_regex ||= disallow_data.empty? ? nil : Regexp.compile(disallow_data.map { |uri| "^#{regex_value(uri)}" }.join("|"))
+  end
+
+  def allow_regex
+    allow_data = @robots_data.fetch(@user_agent, {}).fetch('allow', [])
+    @allow_regex ||= allow_data.empty? ? nil : Regexp.compile(allow_data.map { |uri| "^#{regex_value(uri)}" }.join("|"))
+  end
+
   def check_permission(uri)
     uri = URI.parse(URI.encode(uri))
     return true unless (@robots_data or @robots_data.any?) or (uri.scheme and uri.host)
 
-    uri_path = uri.path.sub(/(\/){2,}$/, '')
-    pattern  = Regexp.compile("(^#{Regexp.escape(uri_path)}[\/]*$)|(^/$)")
-    return (@robots_data.fetch(@user_agent, {}).fetch('disallow', []).grep(pattern).empty? or @robots_data.fetch(@user_agent, {}).fetch('allow', []).grep(pattern).any?)
+    uri_path = strip_slashes(uri.path)
+    return (!!!(uri_path =~ disallow_regex) or !!(uri_path =~ allow_regex))
   end
 
 end

data/robotx.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "robotx"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["Matthias Kalb"]
   spec.email         = ["matthias.kalb@railsmechanic.de"]
   spec.summary       = %q{A parser for the robots.txt file}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: robotx
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Matthias Kalb
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-04 00:00:00.000000000 Z
+date: 2014-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler