fizx-robots 0.3.2 → 0.6.0

data/lib/robots.rb

@@ -1,14 +1,13 @@
 require "open-uri"
 require "uri"
 require "rubygems"
-require "loggable"
+
 class Robots
-  include Loggable
   
   class ParsedRobots
-    include Loggable
     
     def initialize(uri)
+      @last_accessed = Time.at(1)
       io = open(URI.join(uri.to_s, "/robots.txt")) rescue nil
       if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
         io = StringIO.new("User-agent: *\nAllow: /\n")
@@ -17,10 +16,13 @@ class Robots
       @other = {}
       @disallows = {}
       @allows = {}
+      @delays = {} # added delays to make it work
       agent = /.*/
       io.each do |line|
         next if line =~ /^\s*(#.*|$)/
-        key, value = line.split(":")
+        arr = line.split(":")
+        key = arr.shift
+        value = arr.join(":").strip
         value.strip!
         case key
         when "User-agent":
@@ -31,9 +33,10 @@ class Robots
         when "Disallow":
           @disallows[agent] ||= []
           @disallows[agent] << to_regex(value)
+        when "Crawl-delay"
+          @delays[agent] = value.to_i
         else
-          @disallows[agent] ||= []
-          @disallows[agent] << to_regex(value)
+          @other[key] = value
         end
       end
       
@@ -44,38 +47,35 @@ class Robots
       return true unless @parsed
       allowed = true
       path = uri.request_uri
-      debug "path: #{path}"
-      puts "--------"
-      puts @disallows.inspect
-      puts "--------"
+      
       @disallows.each do |key, value|
-        puts ">>>>>>>#{key.inspect}<<<<<<<<"
         if user_agent =~ key
-          debug "matched #{key.inspect}"
           value.each do |rule|
             if path =~ rule
-              debug "matched Disallow: #{rule.inspect}"
               allowed = false
             end
           end
         end
       end
       
-      return true if allowed
-      
       @allows.each do |key, value|
-        if user_agent =~ key
-          debug "matched #{key.inspect}"
-          value.each do |rule|
-            if path =~ rule
-              debug "matched Allow: #{rule.inspect}"
-              return true 
+        unless allowed      
+          if user_agent =~ key
+            value.each do |rule|
+              if path =~ rule
+                allowed = true
+              end
             end
           end
         end
       end
       
-      return false
+      if allowed && @delays[user_agent]
+        sleep @delays[user_agent] - (Time.now - @last_accessed)
+        @last_accessed = Time.now
+      end
+      
+      return allowed
     end
     
     def other_values
@@ -109,4 +109,4 @@ class Robots
     @parsed[host] ||= ParsedRobots.new(uri)
     @parsed[host].other_values
   end
-end
+end

metadata

@@ -1,19 +1,21 @@
 --- !ruby/object:Gem::Specification 
 name: fizx-robots
 version: !ruby/object:Gem::Version 
-  version: 0.3.2
+  version: 0.6.0
 platform: ruby
 authors: 
 - Kyle Maxwell
+- Sausheong Chang
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2008-08-10 00:00:00 -07:00
+date: 2008-12-10 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: fizx-loggable
+  type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: