fizx-fizx-robots 0.4.0

data/README

@@ -0,0 +1,33 @@
+A simple Ruby library to parse robots.txt.
+
+Usage:
+
+	robots = Robots.new "Some User Agent"
+	assert robots.allowed?("http://www.yelp.com/foo")
+	assert !robots.allowed?("http://www.yelp.com/mail?foo=bar")
+	robots.other_values("http://foo.com") # gets misc. key/values (i.e. sitemaps)
+	
+If you want caching, you're on your own.  I suggest marshalling an instance of the parser.  
+
+Copyright (c) 2008 Kyle Maxwell
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/lib/robots.rb

@@ -0,0 +1,118 @@
+require "open-uri"
+require "uri"
+require "rubygems"
+require "loggable"
+class Robots
+  include Loggable
+  
+  class ParsedRobots
+    include Loggable
+    
+    def initialize(uri)
+      @last_accessed = Time.at(1)
+      io = open(URI.join(uri.to_s, "/robots.txt")) rescue nil
+      if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
+        io = StringIO.new("User-agent: *\nAllow: /\n")
+      end
+
+      @other = {}
+      @disallows = {}
+      @allows = {}
+      agent = /.*/
+      io.each do |line|
+        next if line =~ /^\s*(#.*|$)/
+        arr = line.split(":")
+        key = arr.shift
+        value = arr.join(":").strip
+        value.strip!
+        case key
+        when "User-agent":
+          agent = to_regex(value)
+        when "Allow":
+          @allows[agent] ||= []
+          @allows[agent] << to_regex(value)
+        when "Disallow":
+          @disallows[agent] ||= []
+          @disallows[agent] << to_regex(value)
+        when "Crawl-delay"
+          @delays[agent] = value.to_i
+        else
+          @other[key] = value
+        end
+      end
+      
+      @parsed = true
+    end
+    
+    def allowed?(uri, user_agent)
+      return true unless @parsed
+      allowed = true
+      path = uri.request_uri
+      debug "path: #{path}"
+      
+      @disallows.each do |key, value|
+        if user_agent =~ key
+          debug "matched #{key.inspect}"
+          value.each do |rule|
+            if path =~ rule
+              debug "matched Disallow: #{rule.inspect}"
+              allowed = false
+            end
+          end
+        end
+      end
+      
+      @allows.each do |key, value|
+        unless allowed      
+          if user_agent =~ key
+            debug "matched #{key.inspect}"
+            value.each do |rule|
+              if path =~ rule
+                debug "matched Allow: #{rule.inspect}"
+                allowed = true
+              end
+            end
+          end
+        end
+      end
+      
+      if allowed && @delays[user_agent]
+        sleep @delays[user_agent] - (Time.now - @last_accessed)
+        @last_accessed = Time.now
+      end
+      
+      return allowed
+    end
+    
+    def other_values
+      @other
+    end
+    
+  protected
+    
+    def to_regex(pattern)
+      pattern = Regexp.escape(pattern)
+      pattern.gsub!(Regexp.escape("*"), ".*")
+      Regexp.compile("^#{pattern}")
+    end
+  end
+  
+  def initialize(user_agent)
+    @user_agent = user_agent
+    @parsed = {}
+  end
+  
+  def allowed?(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri)
+    @parsed[host].allowed?(uri, @user_agent)
+  end
+  
+  def other_values(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri)
+    @parsed[host].other_values
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification 
+name: fizx-fizx-robots
+version: !ruby/object:Gem::Version 
+  version: 0.4.0
+platform: ruby
+authors: 
+- Kyle Maxwell
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-12-10 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fizx-loggable
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+description: It parses robots.txt files
+email: kyle@kylemaxwell.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- README
+- lib/robots.rb
+has_rdoc: true
+homepage: http://github.com/fizx/robots
+post_install_message: 
+rdoc_options: 
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Simple robots.txt parser
+test_files: []
+