robots 0.8.0 → 0.9.0

data/CHANGELOG

@@ -1,3 +1,6 @@
+0.9.0
+  - Fix http://github.com/fizx/robots/issues#issue/1
+  - Tests don't rely on network.
 0.8.0
   - Add multiple values from robots.txt (via joost)
 0.7.3

data/VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0

data/lib/robots.rb

@@ -12,15 +12,7 @@ class Robots
     def initialize(uri, user_agent)
       @last_accessed = Time.at(1)
       
-      io = nil
-      begin
-        Timeout::timeout(Robots.timeout) do
-          io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
-        end 
-      rescue Timeout::Error
-        STDERR.puts "robots.txt request timed out"
-      end
-        
+      io = Robots.get_robots_txt(uri)
       
       if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
         io = StringIO.new("User-agent: *\nAllow: /\n")
@@ -99,12 +91,23 @@ class Robots
   protected
     
     def to_regex(pattern)
+      return /should-not-match-anything-123456789/ if pattern.strip.empty?
       pattern = Regexp.escape(pattern)
       pattern.gsub!(Regexp.escape("*"), ".*")
       Regexp.compile("^#{pattern}")
     end
   end
   
+  def self.get_robots_txt(uri)
+    begin
+      Timeout::timeout(Robots.timeout) do
+        io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
+      end 
+    rescue Timeout::Error
+      STDERR.puts "robots.txt request timed out"
+    end
+  end
+  
   def self.timeout=(t)
     @timeout = t
   end

data/robots.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{robots}
-  s.version = "0.8.0"
+  s.version = "0.9.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kyle Maxwell"]
-  s.date = %q{2010-02-08}
+  s.date = %q{2010-05-29}
   s.description = %q{It parses robots.txt files}
   s.email = %q{kyle@kylemaxwell.com}
   s.extra_rdoc_files = [
@@ -23,13 +23,17 @@ Gem::Specification.new do |s|
      "VERSION",
      "lib/robots.rb",
      "robots.gemspec",
-     "test/fixtures/robots1.txt",
+     "test/fixtures/emptyish.txt",
+     "test/fixtures/eventbrite.txt",
+     "test/fixtures/google.txt",
+     "test/fixtures/reddit.txt",
+     "test/fixtures/yelp.txt",
      "test/test_robots.rb"
   ]
   s.homepage = %q{http://github.com/fizx/robots}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.5}
+  s.rubygems_version = %q{1.3.6}
   s.summary = %q{Simple robots.txt parser}
   s.test_files = [
     "test/test_robots.rb"

data/test/fixtures/emptyish.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow:

data/test/fixtures/eventbrite.txt

@@ -0,0 +1,435 @@
+# These entries assist in minimizing bandwidth usage caused
+# by questionable robots spidering your site. Some of these
+# robots or agents are used by web-stripping sofware.
+# Please do not remove these entries, but feel free to add
+# your own at the end of the list.
+# If you have any questions regarding this file, please
+# contact support@thinkhost.com
+
+User-agent: *
+Disallow: /rest/
+Disallow: /xml/
+Disallow: /json/
+Disallow: /atom/
+Disallow: /opml/
+Disallow: /widget/
+Disallow: /register
+Disallow: /review
+Disallow: /orderconfirmation
+Disallow: /venues/
+Disallow: /*?
+
+Sitemap: http://www.eventbrite.com/sitemap_index.xml
+
+User-agent: msnbot
+Crawl-delay: 4
+
+User-agent: Slurp
+Crawl-delay: 4
+
+User-agent: Balihoo
+Disallow: /
+
+User-agent: BotRightHere
+Disallow: /
+
+User-agent: WebZip
+Disallow: /
+
+User-agent: larbin
+Disallow: /
+
+User-agent: b2w/0.1
+Disallow: /
+
+User-agent: Copernic
+Disallow: /
+
+User-agent: psbot
+Disallow: /
+
+User-agent: Python-urllib
+Disallow: /
+
+User-agent: NetMechanic
+Disallow: /
+
+User-agent: URL_Spider_Pro
+Disallow: /
+
+User-agent: CherryPicker
+Disallow: /
+
+User-agent: EmailCollector
+Disallow: /
+
+User-agent: EmailSiphon
+Disallow: /
+
+User-agent: WebBandit
+Disallow: /
+
+User-agent: EmailWolf
+Disallow: /
+
+User-agent: ExtractorPro
+Disallow: /
+
+User-agent: CopyRightCheck
+Disallow: /
+
+User-agent: Crescent
+Disallow: /
+
+User-agent: SiteSnagger
+Disallow: /
+
+User-agent: ProWebWalker
+Disallow: /
+
+User-agent: CheeseBot
+Disallow: /
+
+User-agent: LNSpiderguy
+Disallow: /
+
+User-agent: Alexibot
+Disallow: /
+
+User-agent: Teleport
+Disallow: /
+
+User-agent: TeleportPro
+Disallow: /
+
+User-agent: MIIxpc
+Disallow: /
+
+User-agent: Telesoft
+Disallow: /
+
+User-agent: Website Quester
+Disallow: /
+
+User-agent: WebZip
+Disallow: /
+
+User-agent: moget/2.1
+Disallow: /
+
+User-agent: WebZip/4.0
+Disallow: /
+
+User-agent: WebStripper
+Disallow: /
+
+User-agent: WebSauger
+Disallow: /
+
+User-agent: WebCopier
+Disallow: /
+
+User-agent: NetAnts
+Disallow: /
+
+User-agent: Mister PiX
+Disallow: /
+
+User-agent: WebAuto
+Disallow: /
+
+User-agent: TheNomad
+Disallow: /
+
+User-agent: WWW-Collector-E
+Disallow: /
+
+User-agent: RMA
+Disallow: /
+
+User-agent: libWeb/clsHTTP
+Disallow: /
+
+User-agent: asterias
+Disallow: /
+
+User-agent: httplib
+Disallow: /
+
+User-agent: turingos
+Disallow: /
+
+User-agent: spanner
+Disallow: /
+
+User-agent: InfoNaviRobot
+Disallow: /
+
+User-agent: Harvest/1.5
+Disallow: /
+
+User-agent: Bullseye/1.0
+Disallow: /
+
+User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
+Disallow: /
+
+User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
+Disallow: /
+
+User-agent: CherryPickerSE/1.0
+Disallow: /
+
+User-agent: CherryPickerElite/1.0
+Disallow: /
+
+User-agent: WebBandit/3.50
+Disallow: /
+
+User-agent: NICErsPRO
+Disallow: /
+
+User-agent: Microsoft URL Control - 5.01.4511
+Disallow: /
+
+User-agent: DittoSpyder
+Disallow: /
+
+User-agent: Foobot
+Disallow: /
+
+User-agent: SpankBot
+Disallow: /
+
+User-agent: BotALot
+Disallow: /
+
+User-agent: lwp-trivial/1.34
+Disallow: /
+
+User-agent: lwp-trivial
+Disallow: /
+
+User-agent: BunnySlippers
+Disallow: /
+
+User-agent: Microsoft URL Control - 6.00.8169
+Disallow: /
+
+User-agent: URLy Warning
+Disallow: /
+
+User-agent: Wget/1.6
+Disallow: /
+
+User-agent: Wget/1.5.3
+Disallow: /
+
+User-agent: Wget
+Disallow: /
+
+User-agent: LinkWalker
+Disallow: /
+
+User-agent: cosmos
+Disallow: /
+
+User-agent: moget
+Disallow: /
+
+User-agent: hloader
+Disallow: /
+
+User-agent: humanlinks
+Disallow: /
+
+User-agent: LinkextractorPro
+Disallow: /
+
+User-agent: Offline Explorer
+Disallow: /
+
+User-agent: Mata Hari
+Disallow: /
+
+User-agent: LexiBot
+Disallow: /
+
+User-agent: Web Image Collector
+Disallow: /
+
+User-agent: The Intraformant
+Disallow: /
+
+User-agent: True_Robot/1.0
+Disallow: /
+
+User-agent: True_Robot
+Disallow: /
+
+User-agent: BlowFish/1.0
+Disallow: /
+
+User-agent: JennyBot
+Disallow: /
+
+User-agent: MIIxpc/4.2
+Disallow: /
+
+User-agent: BuiltBotTough
+Disallow: /
+
+User-agent: ProPowerBot/2.14
+Disallow: /
+
+User-agent: BackDoorBot/1.0
+Disallow: /
+
+User-agent: toCrawl/UrlDispatcher
+Disallow: /
+
+User-agent: WebEnhancer
+Disallow: /
+
+User-agent: suzuran
+Disallow: /
+
+User-agent: TightTwatBot
+Disallow: /
+
+User-agent: VCI WebViewer VCI WebViewer Win32
+Disallow: /
+
+User-agent: VCI
+Disallow: /
+
+User-agent: Szukacz/1.4
+Disallow: /
+
+User-agent: QueryN Metasearch
+Disallow: /
+
+User-agent: Openfind data gatherer
+Disallow: /
+
+User-agent: Openfind
+Disallow: /
+
+User-agent: Xenu's Link Sleuth 1.1c
+Disallow: /
+
+User-agent: Xenu's
+Disallow: /
+
+User-agent: Zeus
+Disallow: /
+
+User-agent: RepoMonkey Bait & Tackle/v1.01
+Disallow: /
+
+User-agent: RepoMonkey
+Disallow: /
+
+User-agent: Microsoft URL Control
+Disallow: /
+
+User-agent: Openbot
+Disallow: /
+
+User-agent: URL Control
+Disallow: /
+
+User-agent: Zeus Link Scout
+Disallow: /
+
+User-agent: Zeus 32297 Webster Pro V2.9 Win32
+Disallow: /
+
+User-agent: Webster Pro
+Disallow: /
+
+User-agent: EroCrawler
+Disallow: /
+
+User-agent: LinkScan/8.1a Unix
+Disallow: /
+
+User-agent: Keyword Density/0.9
+Disallow: /
+
+User-agent: Kenjin Spider
+Disallow: /
+
+User-agent: Iron33/1.0.2
+Disallow: /
+
+User-agent: Bookmark search tool
+Disallow: /
+
+User-agent: GetRight/4.2
+Disallow: /
+
+User-agent: FairAd Client
+Disallow: /
+
+User-agent: Gaisbot
+Disallow: /
+
+User-agent: Aqua_Products
+Disallow: /
+
+User-agent: Radiation Retriever 1.1
+Disallow: /
+
+User-agent: Flaming AttackBot
+Disallow: /
+
+User-agent: Oracle Ultra Search
+Disallow: /
+
+User-agent: MSIECrawler
+Disallow: /
+
+User-agent: PerMan
+Disallow: /
+
+User-agent: searchpreview
+Disallow: /
+
+User-agent: TurnitinBot
+Disallow: /
+
+User-agent: wget
+Disallow: /
+
+User-agent: ExtractorPro
+Disallow: /
+
+User-agent: WebZIP/4.21
+Disallow: /
+
+User-agent: WebZIP/5.0
+Disallow: /
+
+User-agent: HTTrack 3.0
+Disallow: /
+
+User-agent: TurnitinBot/1.5
+Disallow: /
+
+User-agent: WebCopier v3.2a
+Disallow: /
+
+User-agent: WebCapture 2.0
+Disallow: /
+
+User-agent: WebCopier v.2.2
+Disallow: /
+
+User-agent: Spinn3r
+Disallow: /
+
+User-agent: Tailrank
+Disallow: /
+
+Sitemap: http://www.eventbrite.com/sitemap_index.xml

data/test/fixtures/google.txt

@@ -0,0 +1,215 @@
+User-agent: *
+Disallow: /search
+Disallow: /groups
+Disallow: /images
+Disallow: /catalogs
+Disallow: /catalogues
+Disallow: /news
+Allow: /news/directory
+Disallow: /nwshp
+Disallow: /setnewsprefs?
+Disallow: /index.html?
+Disallow: /?
+Disallow: /addurl/image?
+Disallow: /pagead/
+Disallow: /relpage/
+Disallow: /relcontent
+Disallow: /imgres
+Disallow: /imglanding
+Disallow: /keyword/
+Disallow: /u/
+Disallow: /univ/
+Disallow: /cobrand
+Disallow: /custom
+Disallow: /advanced_group_search
+Disallow: /googlesite
+Disallow: /preferences
+Disallow: /setprefs
+Disallow: /swr
+Disallow: /url
+Disallow: /default
+Disallow: /m?
+Disallow: /m/?
+Disallow: /m/blogs?
+Disallow: /m/ig
+Disallow: /m/images?
+Disallow: /m/local?
+Disallow: /m/movies?
+Disallow: /m/news?
+Disallow: /m/news/i?
+Disallow: /m/place?
+Disallow: /m/setnewsprefs?
+Disallow: /m/search?
+Disallow: /m/swmloptin?
+Disallow: /m/trends
+Disallow: /wml?
+Disallow: /wml/?
+Disallow: /wml/search?
+Disallow: /xhtml?
+Disallow: /xhtml/?
+Disallow: /xhtml/search?
+Disallow: /xml?
+Disallow: /imode?
+Disallow: /imode/?
+Disallow: /imode/search?
+Disallow: /jsky?
+Disallow: /jsky/?
+Disallow: /jsky/search?
+Disallow: /pda?
+Disallow: /pda/?
+Disallow: /pda/search?
+Disallow: /sprint_xhtml
+Disallow: /sprint_wml
+Disallow: /pqa
+Disallow: /palm
+Disallow: /gwt/
+Disallow: /purchases
+Disallow: /hws
+Disallow: /bsd?
+Disallow: /linux?
+Disallow: /mac?
+Disallow: /microsoft?
+Disallow: /unclesam?
+Disallow: /answers/search?q=
+Disallow: /local?
+Disallow: /local_url
+Disallow: /froogle?
+Disallow: /products?
+Disallow: /products/
+Disallow: /froogle_
+Disallow: /product_
+Disallow: /products_
+Disallow: /print
+Disallow: /books
+Disallow: /bkshp?q=
+Allow: /booksrightsholders
+Disallow: /patents?
+Disallow: /patents/
+Allow: /patents/about
+Disallow: /scholar
+Disallow: /complete
+Disallow: /sponsoredlinks
+Disallow: /videosearch?
+Disallow: /videopreview?
+Disallow: /videoprograminfo?
+Disallow: /maps?
+Disallow: /mapstt?
+Disallow: /mapslt?
+Disallow: /maps/stk/
+Disallow: /maps/br?
+Disallow: /mapabcpoi?
+Disallow: /maphp?
+Disallow: /places/
+Disallow: /maps/place
+Disallow: /help/maps/streetview/partners/welcome/
+Disallow: /lochp?
+Disallow: /center
+Disallow: /ie?
+Disallow: /sms/demo?
+Disallow: /katrina?
+Disallow: /blogsearch?
+Disallow: /blogsearch/
+Disallow: /blogsearch_feeds
+Disallow: /advanced_blog_search
+Disallow: /reader/
+Allow: /reader/play
+Disallow: /uds/
+Disallow: /chart?
+Disallow: /transit?
+Disallow: /mbd?
+Disallow: /extern_js/
+Disallow: /calendar/feeds/
+Disallow: /calendar/ical/
+Disallow: /cl2/feeds/
+Disallow: /cl2/ical/
+Disallow: /coop/directory
+Disallow: /coop/manage
+Disallow: /trends?
+Disallow: /trends/music?
+Disallow: /trends/hottrends?
+Disallow: /trends/viz?
+Disallow: /notebook/search?
+Disallow: /musica
+Disallow: /musicad
+Disallow: /musicas
+Disallow: /musicl
+Disallow: /musics
+Disallow: /musicsearch
+Disallow: /musicsp
+Disallow: /musiclp
+Disallow: /browsersync
+Disallow: /call
+Disallow: /archivesearch?
+Disallow: /archivesearch/url
+Disallow: /archivesearch/advanced_search
+Disallow: /base/reportbadoffer
+Disallow: /urchin_test/
+Disallow: /movies?
+Disallow: /codesearch?
+Disallow: /codesearch/feeds/search?
+Disallow: /wapsearch?
+Disallow: /safebrowsing
+Allow: /safebrowsing/diagnostic
+Allow: /safebrowsing/report_error/
+Allow: /safebrowsing/report_phish/
+Disallow: /reviews/search?
+Disallow: /orkut/albums
+Disallow: /jsapi
+Disallow: /views?
+Disallow: /c/
+Disallow: /cbk
+Disallow: /recharge/dashboard/car
+Disallow: /recharge/dashboard/static/
+Disallow: /translate_a/
+Disallow: /translate_c
+Disallow: /translate_f
+Disallow: /translate_static/
+Disallow: /translate_suggestion
+Disallow: /profiles/me
+Allow: /profiles
+Disallow: /s2/profiles/me
+Allow: /s2/profiles
+Allow: /s2/photos
+Allow: /s2/static
+Disallow: /s2
+Disallow: /transconsole/portal/
+Disallow: /gcc/
+Disallow: /aclk
+Disallow: /cse?
+Disallow: /cse/home
+Disallow: /cse/panel
+Disallow: /cse/manage
+Disallow: /tbproxy/
+Disallow: /comparisonads/
+Disallow: /imesync/
+Disallow: /shenghuo/search?
+Disallow: /support/forum/search?
+Disallow: /reviews/polls/
+Disallow: /hosted/images/
+Disallow: /ppob/?
+Disallow: /ppob?
+Disallow: /ig/add?
+Disallow: /adwordsresellers
+Disallow: /accounts/o8
+Allow: /accounts/o8/id
+Disallow: /topicsearch?q=
+Disallow: /xfx7/
+Disallow: /squared/api
+Disallow: /squared/search
+Disallow: /squared/table
+Disallow: /toolkit/
+Allow: /toolkit/*.html
+Disallow: /qnasearch?
+Disallow: /errors/
+Disallow: /app/updates
+Disallow: /sidewiki/entry/
+Disallow: /quality_form?
+Disallow: /labs/popgadget/search
+Disallow: /buzz/post
+Disallow: /compressiontest/
+Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml
+Sitemap: http://www.google.com/hostednews/sitemap_index.xml
+Sitemap: http://www.google.com/ventures/sitemap_ventures.xml
+Sitemap: http://www.google.com/sitemaps_webmasters.xml
+Sitemap: http://www.gstatic.com/trends/websites/sitemaps/sitemapindex.xml
+Sitemap: http://www.gstatic.com/dictionary/static/sitemaps/sitemap_index.xml

data/test/fixtures/reddit.txt

@@ -0,0 +1,14 @@
+# 80legs
+User-agent: 008
+Disallow: /
+
+User-Agent: *
+Disallow: /goto
+Disallow: /*after=
+Disallow: /*before=
+Disallow: /domain/*t=
+Disallow: /login
+Disallow: /reddits/search
+Disallow: /search
+Disallow: /r/*/search
+Allow: /

data/test/fixtures/yelp.txt

@@ -0,0 +1,68 @@
+#
+# 1. A robot may not injure a human being or, through inaction, allow a
+# human being to come to harm.
+#
+# 2. A robot must obey orders given it by human beings except where such
+# orders would conflict with the First Law.
+#
+# 3. A robot must protect its own existence as long as such protection
+# does not conflict with the First or Second Law.
+
+User-agent: *
+Disallow: /advertise?
+Disallow: /biz_share?
+Disallow: /biz_attribute
+Disallow: /biz_link
+Disallow: /biz_update
+Disallow: /bookmark?
+Disallow: /flag_content?
+Disallow: /invite_friends_service?
+Disallow: /login?
+Disallow: /mail?
+Disallow: /map?
+Disallow: /redir?
+Disallow: /writeareview
+Disallow: /signup?
+Disallow: /talk/new_topic
+Disallow: /thanx?
+Disallow: /user_favorites?
+Disallow: /weekly/signup
+Disallow: /elite?
+Disallow: /member_search_results
+Disallow: /advertise?
+Disallow: /syndicate/
+Disallow: /filtered_reviews
+Disallow: /language/update?
+
+User-agent: Fasterfox
+Disallow: /
+
+User-agent: Nutch
+Disallow: /
+
+User-agent: spock
+Disallow: /
+
+User-agent: OmniExplorer_Bot
+Disallow: /
+
+User-agent: MJ12bot
+Disallow: /
+
+User-agent: TurnitinBot
+Disallow: /
+
+User-agent: BecomeBot 
+Disallow: /
+
+User-agent: genieBot
+Disallow: /
+
+User-agent: dotbot 
+Disallow: / 
+
+User-agent: MLBot
+Disallow: /
+
+User-agent: 80bot
+Disallow: /

data/test/test_robots.rb

@@ -2,44 +2,75 @@
 require "test/unit"
 require File.dirname(__FILE__) + "/../lib/robots"
 
-module Kernel
-  alias_method :open_old, :open
-  
-  def set_open(key, value)
-    @fake_open_values ||= {}
-    @fake_open_values[key] = value
+module FakeHttp
+  def content_type
+    "text/plain"
   end
   
-  def open(*args)
-    @fake_open_values ||= {}
-    @fake_open_values[args.first] || open_old(*args)
+  def status
+    ["200", "OK"]
   end
 end
 
 class TestRobots < Test::Unit::TestCase
   def setup
+    def Robots.get_robots_txt(uri)
+      fixture_file = File.dirname(__FILE__) + "/fixtures/" + uri.host.split(".")[-2] + ".txt"
+      File.open(fixture_file).extend(FakeHttp)
+    end
+    
     @robots = Robots.new "Ruby-Robot.txt Parser Test Script"
   end
   
   def test_allowed_if_no_robots
-    assert @robots.allowed?("http://www.yahoo.com")
+    def Robots.get_robots_txt(uri)
+      return nil
+    end
+    
+    assert_allowed("somesite", "/")
+  end
+  
+  def test_disallow_nothing
+    assert_allowed("emptyish", "/")
+    assert_allowed("emptyish", "/foo")
   end
   
   def test_reddit
-    assert @robots.allowed?("http://reddit.com")
+    assert_allowed("reddit", "/")
   end
   
   def test_other
-    assert @robots.allowed?("http://www.yelp.com/foo")
-    assert !@robots.allowed?("http://www.yelp.com/mail?foo=bar")
+    assert_allowed("yelp", "/foo")
+    assert_disallowed("yelp", "/mail?foo=bar")
   end
   
   def test_site_with_disallowed
-    assert @robots.allowed?("http://www.google.com/")
+    assert_allowed("google", "/")
   end
   
   def test_other_values
     sitemap = {"Sitemap" => ["http://www.eventbrite.com/sitemap_index.xml", "http://www.eventbrite.com/sitemap_index.xml"]}
-    assert_equal(sitemap, @robots.other_values("http://eventbrite.com"))
+    assert_other_equals("eventbrite", sitemap)
+  end
+  
+  def assert_other_equals(name, value)
+    assert_equal(value, @robots.other_values(uri_for_name(name, "/")))
+  end
+  
+  def assert_allowed(name, path)
+    assert_allowed_equals(name, path, true)
+  end
+  
+  def assert_disallowed(name, path)
+    assert_allowed_equals(name, path, false)
+  end
+  
+  def assert_allowed_equals(name, path, value)
+    assert_equal(value, @robots.allowed?(uri_for_name(name, path)), @robots.inspect)
+  end
+  
+  def uri_for_name(name, path)
+    uri = name.nil? ? nil : "http://www.#{name}.com#{path}"
   end
+    
 end

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: robots
 version: !ruby/object:Gem::Version 
-  version: 0.8.0
+  prerelease: false
+  segments: 
+  - 0
+  - 9
+  - 0
+  version: 0.9.0
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -9,19 +14,21 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-08 00:00:00 -08:00
+date: 2010-05-29 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: thoughtbot-shoulda
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :development
+  version_requirements: *id001
 description: It parses robots.txt files
 email: kyle@kylemaxwell.com
 executables: []
@@ -38,7 +45,11 @@ files:
 - VERSION
 - lib/robots.rb
 - robots.gemspec
-- test/fixtures/robots1.txt
+- test/fixtures/emptyish.txt
+- test/fixtures/eventbrite.txt
+- test/fixtures/google.txt
+- test/fixtures/reddit.txt
+- test/fixtures/yelp.txt
 - test/test_robots.rb
 has_rdoc: true
 homepage: http://github.com/fizx/robots
@@ -53,18 +64,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Simple robots.txt parser