simply-robotstxt 1.0.4

data/README

@@ -0,0 +1,22 @@
+robots.txt is an extremely simply format, but it still needs some parser loving. This class will normalise robots.txt entries for you.
+
+USAGE
+-----
+
+With the following robots.txt:
+
+User-agent: *
+Disallow: /logs
+
+User-agent: Google
+Disallow: /admin
+
+Use it like this:
+
+require 'robotstxtparser'
+
+# Also accepts a local file
+rp = RobotsTxtParser.new()
+rp.read("http://something.com/robots.txt")
+
+rp.user_agents('Google') # returns ["/logs", "/admin"]

data/lib/robotstxtparser.rb

@@ -0,0 +1,48 @@
+require 'open-uri'
+
+class RobotsTxtParser
+
+  attr_reader :user_agents
+
+  def read(path)
+    begin
+      if path.include?("://")
+        raw_data = open(path)
+      else
+          raw_data = File.open(path)
+      end
+    rescue
+      raw_data = nil
+    end
+
+    @user_agents = Hash.new
+
+    return unless raw_data
+
+    parse(raw_data)
+  end
+
+  def parse(raw_data)
+    current_agent = nil
+
+    raw_data.each_line do |line|
+
+      if line.match(/^User Agent:/)
+        current_agent = line.gsub("User Agent:","").strip
+      elsif line.match(/^Disallow:/)
+	@user_agents[current_agent] = Array.new unless @user_agents[current_agent]
+        @user_agents[current_agent].push line.gsub("Disallow:", "").strip
+      end
+    end
+
+    add_wildcard_records 
+  end
+
+  def add_wildcard_records
+    if @user_agents.has_key?('*')
+      @user_agents.each do |agent, records|
+        @user_agents[agent] = records + @user_agents['*']
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification 
+name: simply-robotstxt
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 4
+  version: 1.0.4
+platform: ruby
+authors: 
+- Colin Ramsay
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-06-30 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Parse robots.txt files from Ruby.
+email: colin@gotripod.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- lib/robotstxtparser.rb
+has_rdoc: true
+homepage: http://github.com/colinramsay/simply-robotstxt
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Parse robots.txt files from Ruby.
+test_files: []
+