robotex 1.0.0

data/CHANGELOG.rdoc

@@ -0,0 +1,12 @@
+== 1.0.0 / 2012-01-20
+
+* Minor enhancements
+
+  * Move delay sleep out of allowed? into its own method (delay!)
+  * Add rspec suite
+  * Add Gemfile
+
+* Bug fixes
+
+  * Fix handling of priority for allow vs disallow
+  * Fix permission issue with gem

data/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2012 Chris Kite
+Copyright (c) 2008-2011 Kyle Maxwell, contributors
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+

data/README.rdoc

@@ -0,0 +1,14 @@
+= Robotex
+== Obey Robots.txt
+
+With one line of code, Robotex (pronounced like "robotics") will download and parse the robots.txt file and let you know if your program is allowed to visit a given link.
+
+Usage:
+
+    robotex = Robotex.new "My User Agent"
+    robotex.allowed?("http://www.example.com/foo")
+    robotex.delay! # wait until any specified Crawl-Delay has passed
+
+== Acknowledgements
+
+Robotex is a modified version of Kyle Maxwell's excellent Robots library. Some folks were unable to use that gem due to packaging issues, so I used his code to create Robotex.

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rspec/core/rake_task'
+require 'rake/rdoctask'
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:rspec) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :rspec
+
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "Robotex #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/robotex.rb

@@ -0,0 +1,152 @@
+# Author (2012): Chris Kite
+# Original Author (2008-2011): Kyle Maxwell
+
+require 'rubygems'
+require 'open-uri'
+require 'uri'
+require 'timeout'
+
+class Robotex
+  
+  VERSION = '1.0.0'
+  DEFAULT_TIMEOUT = 3
+
+  attr_reader :user_agent
+  
+  class ParsedRobots
+    
+    def initialize(uri, user_agent)
+      io = Robotex.get_robots_txt(uri, user_agent)
+      
+      if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
+        io = StringIO.new("User-agent: *\nAllow: /\n")
+      end
+
+      @disallows = {}
+      @allows = {}
+      @delays = {}
+      agent = /.*/
+      io.each do |line|
+        next if line =~ /^\s*(#.*|$)/
+        arr = line.split(":")
+        key = arr.shift
+        value = arr.join(":").strip
+        value.strip!
+        case key.downcase
+          when "user-agent"
+            agent = to_regex(value)
+          when "allow"
+            @allows[agent] ||= []
+            @allows[agent] << to_regex(value)
+          when "disallow"
+            @disallows[agent] ||= []
+            @disallows[agent] << to_regex(value)
+          when "crawl-delay"
+            @delays[agent] = value.to_i
+        end
+      end
+      
+      @parsed = true
+    end
+    
+    def allowed?(uri, user_agent)
+      return true unless @parsed
+      allowed = true
+      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+      path = uri.request_uri
+      
+      @allows.each do |key, value|
+        unless allowed      
+          if user_agent =~ key
+            value.each do |rule|
+              if path =~ rule
+                allowed = true
+              end
+            end
+          end
+        end
+      end
+      
+      @disallows.each do |key, value|
+        if user_agent =~ key
+          value.each do |rule|
+            if path =~ rule
+              allowed = false
+            end
+          end
+        end
+      end
+      
+      return allowed
+    end
+
+    def delay(user_agent)
+      @delays.each do |agent, delay|
+        return delay if agent =~ user_agent
+      end
+      nil
+    end
+    
+    protected
+    
+    def to_regex(pattern)
+      return /should-not-match-anything-123456789/ if pattern.strip.empty?
+      pattern = Regexp.escape(pattern)
+      pattern.gsub!(Regexp.escape("*"), ".*")
+      Regexp.compile("^#{pattern}")
+    end
+  end
+  
+  def self.get_robots_txt(uri, user_agent)
+    begin
+      Timeout::timeout(Robotex.timeout) do
+        io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
+      end 
+    rescue Timeout::Error
+      STDERR.puts "robots.txt request timed out"
+    end
+  end
+  
+  def self.timeout=(t)
+    @timeout = t
+  end
+  
+  def self.timeout
+    @timeout || DEFAULT_TIMEOUT
+  end
+
+  def initialize(user_agent = nil)
+    user_agent = "Robotex/#{VERSION} (http://www.github.com/chriskite/robotex)" if user_agent.nil?
+    @user_agent = user_agent
+    @last_accessed = Time.at(1)
+    @parsed = {}
+  end
+
+  def parse_host(uri)
+    uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+    @parsed[uri.host] ||= ParsedRobots.new(uri, @user_agent)
+  end
+  
+  #
+  # Download the server's robots.txt, and return try if we are allowed to acces the url, false otherwise
+  #
+  def allowed?(uri)
+    parse_host(uri).allowed?(uri, @user_agent)
+  end
+
+  #
+  # Return the value of the Crawl-Delay directive, or nil if none
+  def delay(uri)
+    parse_host(uri).delay(@user_agent)
+  end
+
+  #
+  # Sleep for the amount of time necessary to obey the Crawl-Delay specified by the server
+  #
+  def delay!(uri)
+    delay = delay(uri)
+    sleep delay - (Time.now - @last_accessed) if !!delay
+    @last_accessed = Time.now
+  end
+  
+end

data/spec/robotex_spec.rb

@@ -0,0 +1,87 @@
+require 'spec_helper'
+
+describe Robotex do
+
+  before(:all) do
+    FakeWeb.allow_net_connect = false
+    robots = <<-END
+User-Agent: msnbot
+Crawl-Delay: 20
+
+User-Agent: bender
+Disallow: /my_shiny_metal_ass
+
+User-Agent: *
+Disallow: /login
+Allow: /
+
+Disallow: /locked
+Allow: /locked
+END
+    options = {:body => robots, :content_type => 'text/plain', :status => [200, "OK"]}
+    FakeWeb.register_uri(:get, SPEC_DOMAIN + 'robots.txt', options)
+  end
+
+  describe '#initialize' do
+    context 'when no arguments are supplied' do
+      it 'returns a Robotex with the default user-agent' do
+        Robotex.new.user_agent.should == "Robotex/#{Robotex::VERSION} (http://www.github.com/chriskite/robotex)"
+      end
+    end
+
+    context 'when a user-agent is specified' do
+      it 'returns a Robotex with the specified user-agent' do
+        ua = 'My User Agent'
+        Robotex.new(ua).user_agent.should == ua
+      end
+    end
+  end
+
+  describe '#allowed?' do
+    context 'when the robots.txt disallows the user-agent to the url' do
+      it 'returns false' do
+        robotex = Robotex.new('bender')
+        robotex.allowed?(SPEC_DOMAIN + 'my_shiny_metal_ass').should be_false
+      end
+    end
+
+    context 'when the robots.txt disallows the user-agent to some urls, but allows this one' do
+      it 'returns true' do
+        robotex = Robotex.new('bender')
+        robotex.allowed?(SPEC_DOMAIN + 'cigars').should be_true
+      end
+    end
+
+    context 'when the robots.txt disallows any user-agent to the url' do
+      it 'returns false' do
+        robotex = Robotex.new
+        robotex.allowed?(SPEC_DOMAIN + 'login').should be_false
+      end
+    end
+
+    context 'when the robots.txt disallows and then allows the url' do
+      it 'returns false' do
+        robotex = Robotex.new
+        robotex.allowed?(SPEC_DOMAIN + 'locked').should be_false
+      end
+    end
+  end
+
+  describe '#delay' do
+    context 'when no Crawl-Delay is specified for the user-agent' do
+      it 'returns nil' do
+        robotex = Robotex.new
+        robotex.delay(SPEC_DOMAIN).should be_nil 
+      end
+
+    context 'when Crawl-Delay is specified for the user-agent' do
+      it 'returns the delay as a Fixnum' do
+        robotex = Robotex.new('msnbot')
+        robotex.delay(SPEC_DOMAIN).should == 20
+      end
+    end
+    end
+  end
+
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'bundler/setup'
+$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'robotex'
+require 'fakeweb'
+
+SPEC_DOMAIN = 'http://www.example.com/'
+

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: robotex
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Chris Kite
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-20 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &21394420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *21394420
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: &21393840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: *21393840
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &21393260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :development
+  prerelease: false
+  version_requirements: *21393260
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: &21392680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *21392680
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+files:
+- VERSION
+- LICENSE
+- CHANGELOG.rdoc
+- README.rdoc
+- Rakefile
+- lib/robotex.rb
+- spec/spec_helper.rb
+- spec/robotex_spec.rb
+homepage: http://www.github.com/chriskite/robotex
+licenses: []
+post_install_message: 
+rdoc_options:
+- -m
+- README.rdoc
+- -t
+- Robotex
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Obey Robots.txt
+test_files:
+- spec/spec_helper.rb
+- spec/robotex_spec.rb