fizx-robots 0.1.1

data/README

@@ -0,0 +1,33 @@
+A simple Ruby library to parse robots.txt.
+
+Usage:
+
+	robots = Robots.new "Some User Agent"
+	assert robots.allowed?("http://www.yelp.com/foo")
+	assert !robots.allowed?("http://www.yelp.com/mail?foo=bar")
+	robots.other_values("http://foo.com") # gets misc. key/values (i.e. sitemaps)
+	
+If you want caching, you're on your own.  I suggest marshalling an instance of the parser.  
+
+Copyright (c) 2008 Kyle Maxwell
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/lib/robots.rb

@@ -0,0 +1,113 @@
+require "open-uri"
+require "uri"
+class Robots
+  class ParsedRobots
+    def initialize(uri)
+      io = open(URI.join(uri.to_s, "/robots.txt"))
+      return if io.content_type != "text/plain"
+      return if io.status != ["200", "OK"]
+
+      @other = {}
+      @disallows = {}
+      @allows = {}
+      agent = ""
+      io.each do |line|
+        next if line =~ /^\s*(#.*|$)/
+        key, value = line.split(":")
+        value.strip!
+        case key
+        when "User-agent":
+          agent = to_regex(value)
+        when "Allow":
+          @allows[agent] ||= []
+          @allows[agent] << to_regex(value)
+        when "Disallow":
+          @disallows[agent] ||= []
+          @disallows[agent] << to_regex(value)
+        else
+          @disallows[key] ||= []
+          @disallows[key] << value
+        end
+      end
+      
+      @parsed = true
+    end
+    
+    def allowed?(uri, user_agent)
+      return true unless @parsed
+      allowed = true
+      path = uri.request_uri
+      puts "path: #{path}"
+      
+      @disallows.each do |key, value|
+        if user_agent =~ key
+          puts "matched #{key.inspect}"
+          value.each do |rule|
+            if path =~ rule
+              puts "matched Disallow: #{rule.inspect}"
+              allowed = false
+            end
+          end
+        end
+      end
+      
+      return true if allowed
+      
+      @allows.each do |key, value|
+        if user_agent =~ key
+          puts "matched #{key.inspect}"
+          value.each do |rule|
+            if path =~ rule
+              puts "matched Allow: #{rule.inspect}"
+              return true 
+            end
+          end
+        end
+      end
+      
+      return false
+    end
+    
+    def other_values
+      @other
+    end
+    
+  protected
+    
+    def to_regex(pattern)
+      pattern = Regexp.escape(pattern)
+      pattern.gsub!(Regexp.escape("*"), ".*")
+      Regexp.compile("^#{pattern}")
+    end
+  end
+  
+  def initialize(user_agent)
+    @user_agent = user_agent
+    @parsed = {}
+  end
+  
+  def allowed?(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri)
+    @parsed[host].allowed?(uri, @user_agent)
+  end
+  
+  def other_values(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri)
+    @parsed[host].other_values
+  end
+end
+
+if __FILE__ == $0
+  require "test/unit"
+  class RobotsTest < Test::Unit::TestCase 
+    def test_robots
+      robots = Robots.new "Ruby-Robot.txt Parser Test Script"
+      assert robots.allowed?("http://www.yelp.com/foo")
+      assert !robots.allowed?("http://www.yelp.com/mail?foo=bar")
+    end
+  end  
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification 
+name: fizx-robots
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Kyle Maxwell
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-08-10 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: It parses robots.txt files
+email: kyle@kylemaxwell.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- README
+- lib/robots.rb
+has_rdoc: true
+homepage: http://github.com/fizx/robots
+post_install_message: 
+rdoc_options: 
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Simple robots.txt parser
+test_files: []
+