robots 0.0.0 → 0.7.3

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/CHANGELOG

@@ -0,0 +1,19 @@
+0.7.3
+  - Move to jeweler
+0.7.2
+  - Add Ruby 1.9 compatibility
+0.5-0.7.1
+  - Lost the changelog information :/
+0.4.0 
+  - Fixed other_values bug
+  - added crawl-delay support
+0.3.2
+	- fixed breaking on reddit.com
+0.3.1
+  - fixed bug in disallows handling
+  - partially mocked out open-uri
+0.3.0
+  - added loggable dependency 
+0.2.0
+	- IF robot.txt 404s, assume allowed.
+  - Added CHANGELOG

data/README

@@ -0,0 +1,33 @@
+A simple Ruby library to parse robots.txt.
+
+Usage:
+
+	robots = Robots.new "Some User Agent"
+	assert robots.allowed?("http://www.yelp.com/foo")
+	assert !robots.allowed?("http://www.yelp.com/mail?foo=bar")
+	robots.other_values("http://foo.com") # gets misc. key/values (i.e. sitemaps)
+	
+If you want caching, you're on your own.  I suggest marshalling an instance of the parser.  
+
+Copyright (c) 2008 Kyle Maxwell
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,57 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "robots"
+    gem.summary = "Simple robots.txt parser"
+    gem.description = "It parses robots.txt files"
+    gem.email = "kyle@kylemaxwell.com"
+    gem.homepage = "http://github.com/fizx/robots"
+    gem.authors = ["Kyle Maxwell"]
+    gem.add_development_dependency "thoughtbot-shoulda"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "robots #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.7.3

data/lib/robots.rb

@@ -0,0 +1,133 @@
+require "open-uri"
+require "uri"
+require "rubygems"
+require "timeout"
+
+class Robots
+  
+  DEFAULT_TIMEOUT = 3
+  
+  class ParsedRobots
+    
+    def initialize(uri, user_agent)
+      @last_accessed = Time.at(1)
+      
+      io = nil
+      begin
+        Timeout::timeout(Robots.timeout) do
+          io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
+        end 
+      rescue Timeout::Error
+        STDERR.puts "robots.txt request timed out"
+      end
+        
+      
+      if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
+        io = StringIO.new("User-agent: *\nAllow: /\n")
+      end
+
+      @other = {}
+      @disallows = {}
+      @allows = {}
+      @delays = {} # added delays to make it work
+      agent = /.*/
+      io.each do |line|
+        next if line =~ /^\s*(#.*|$)/
+        arr = line.split(":")
+        key = arr.shift
+        value = arr.join(":").strip
+        value.strip!
+        case key
+        when "User-agent"
+          agent = to_regex(value)
+        when "Allow"
+          @allows[agent] ||= []
+          @allows[agent] << to_regex(value)
+        when "Disallow"
+          @disallows[agent] ||= []
+          @disallows[agent] << to_regex(value)
+        when "Crawl-delay"
+          @delays[agent] = value.to_i
+        else
+          @other[key] = value
+        end
+      end
+      
+      @parsed = true
+    end
+    
+    def allowed?(uri, user_agent)
+      return true unless @parsed
+      allowed = true
+      path = uri.request_uri
+      
+      @disallows.each do |key, value|
+        if user_agent =~ key
+          value.each do |rule|
+            if path =~ rule
+              allowed = false
+            end
+          end
+        end
+      end
+      
+      @allows.each do |key, value|
+        unless allowed      
+          if user_agent =~ key
+            value.each do |rule|
+              if path =~ rule
+                allowed = true
+              end
+            end
+          end
+        end
+      end
+      
+      if allowed && @delays[user_agent]
+        sleep @delays[user_agent] - (Time.now - @last_accessed)
+        @last_accessed = Time.now
+      end
+      
+      return allowed
+    end
+    
+    def other_values
+      @other
+    end
+    
+  protected
+    
+    def to_regex(pattern)
+      pattern = Regexp.escape(pattern)
+      pattern.gsub!(Regexp.escape("*"), ".*")
+      Regexp.compile("^#{pattern}")
+    end
+  end
+  
+  def self.timeout=(t)
+    @timeout = t
+  end
+  
+  def self.timeout
+    @timeout || DEFAULT_TIMEOUT
+  end
+  
+  def initialize(user_agent)
+    @user_agent = user_agent
+    @parsed = {}
+  end
+  
+  def allowed?(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri, @user_agent)
+    @parsed[host].allowed?(uri, @user_agent)
+  end
+  
+  def other_values(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    host = uri.host
+    @parsed[host] ||= ParsedRobots.new(uri, @user_agent)
+    @parsed[host].other_values
+  end
+end

data/robots.gemspec

@@ -0,0 +1,50 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{robots}
+  s.version = "0.7.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Kyle Maxwell"]
+  s.date = %q{2009-10-18}
+  s.description = %q{It parses robots.txt files}
+  s.email = %q{kyle@kylemaxwell.com}
+  s.extra_rdoc_files = [
+    "README"
+  ]
+  s.files = [
+    ".gitignore",
+     "CHANGELOG",
+     "README",
+     "Rakefile",
+     "VERSION",
+     "lib/robots.rb",
+     "robots.gemspec",
+     "test/fixtures/robots1.txt",
+     "test/test_robots.rb"
+  ]
+  s.homepage = %q{http://github.com/fizx/robots}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{Simple robots.txt parser}
+  s.test_files = [
+    "test/test_robots.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    else
+      s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+  end
+end

data/test/test_robots.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+require "test/unit"
+require File.dirname(__FILE__) + "/../lib/robots"
+
+module Kernel
+  alias_method :open_old, :open
+  
+  def set_open(key, value)
+    @fake_open_values ||= {}
+    @fake_open_values[key] = value
+  end
+  
+  def open(*args)
+    @fake_open_values ||= {}
+    @fake_open_values[args.first] || open_old(*args)
+  end
+end
+
+class TestRobots < Test::Unit::TestCase
+  def setup
+    @robots = Robots.new "Ruby-Robot.txt Parser Test Script"
+  end
+  
+  def test_allowed_if_no_robots
+    assert @robots.allowed?("http://www.yahoo.com")
+  end
+  
+  def test_reddit
+    assert @robots.allowed?("http://reddit.com")
+  end
+  
+  def test_other
+    assert @robots.allowed?("http://www.yelp.com/foo")
+    assert !@robots.allowed?("http://www.yelp.com/mail?foo=bar")
+  end
+  
+  def test_site_with_disallowed
+    assert @robots.allowed?("http://www.google.com/")
+  end
+  
+  def test_other_values
+    sitemap = {"Sitemap" => "http://www.eventbrite.com/sitemap_index.xml"}
+    assert_equal(sitemap, @robots.other_values("http://eventbrite.com"))
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: robots
 version: !ruby/object:Gem::Version 
-  version: 0.0.0
+  version: 0.7.3
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -9,27 +9,44 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-10 00:00:00 -07:00
+date: 2009-10-18 00:00:00 -07:00
 default_executable: 
-dependencies: []
-
-description: 
-email: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: It parses robots.txt files
+email: kyle@kylemaxwell.com
 executables: []
 
 extensions: []
 
-extra_rdoc_files: []
-
-files: []
-
+extra_rdoc_files: 
+- README
+files: 
+- .gitignore
+- CHANGELOG
+- README
+- Rakefile
+- VERSION
+- lib/robots.rb
+- robots.gemspec
+- test/fixtures/robots1.txt
+- test/test_robots.rb
 has_rdoc: true
-homepage: 
+homepage: http://github.com/fizx/robots
 licenses: []
 
 post_install_message: 
-rdoc_options: []
-
+rdoc_options: 
+- --charset=UTF-8
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -50,6 +67,6 @@ rubyforge_project:
 rubygems_version: 1.3.4
 signing_key: 
 specification_version: 3
-summary: Placeholder for a gem to be migrated later
-test_files: []
-
+summary: Simple robots.txt parser
+test_files: 
+- test/test_robots.rb