fizx-robots 0.1.1 → 0.2.0

data/lib/robots.rb

@@ -3,9 +3,10 @@ require "uri"
 class Robots
   class ParsedRobots
     def initialize(uri)
-      io = open(URI.join(uri.to_s, "/robots.txt"))
-      return if io.content_type != "text/plain"
-      return if io.status != ["200", "OK"]
+      io = open(URI.join(uri.to_s, "/robots.txt")) rescue nil
+      if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
+        io = StringIO.new("User-agent: *\nAllow: /\n")
+      end
 
       @other = {}
       @disallows = {}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: fizx-robots
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors: 
 - Kyle Maxwell