colinramsay-robotstxt 1.0

data/README

@@ -0,0 +1,17 @@
+robots.txt is an extremely simply format, but it still needs some parser loving. This class will normalise robots.txt entries for you.
+
+USAGE
+-----
+
+With the following robots.txt:
+
+User-agent: *
+Disallow: /logs
+
+User-agent: Google
+Disallow: /admin
+
+# Also accepts a local file
+rp = RobotsParser.new("http://something.com/robots.txt")
+
+rp.user_agents('Google') # returns ["/logs", "/admin"]

data/lib/robotstxtparser.rb

@@ -0,0 +1,44 @@
+require 'open-uri'
+
+class RobotsTxtParser
+
+  attr_reader :user_agents
+
+  def initialize(path)
+    if path.include?("://")
+      raw_data = open(path)
+    else
+      raw_data = File.open(path)
+    end
+
+    return unless raw_data
+
+    @user_agents = Hash.new
+
+    parse(raw_data)
+  end
+
+  def parse(raw_data)
+    current_agent = nil
+
+    raw_data.each_line do |line|
+
+      if line.match(/^User Agent:/)
+        current_agent = line.gsub("User Agent:","").strip
+      elsif line.match(/^Disallow:/)
+	@user_agents[current_agent] = Array.new unless @user_agents[current_agent]
+        @user_agents[current_agent].push line.gsub("Disallow:", "").strip
+      end
+    end
+
+    add_wildcard_records 
+  end
+
+  def add_wildcard_records
+    if @user_agents.has_key?('*')
+      @user_agents.each do |agent, records|
+        @user_agents[agent] = records + @user_agents['*']
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification 
+name: colinramsay-robotstxt
+version: !ruby/object:Gem::Version 
+  version: "1.0"
+platform: ruby
+authors: 
+- Colin Ramsay
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-06-23 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Parse robots.txt files from Ruby.
+email: colin@gotripod.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- lib/robotstxtparser.rb
+has_rdoc: false
+homepage: http://github.com/colinramsay/robotstxt
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Parse robots.txt files from Ruby.
+test_files: []
+