webrobots 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 897c27b67d9ebd1b6e4ab138725222489b1e7c27
-  data.tar.gz: 23ea1e0155c11932841724b3379d8baf3d372e75
+  metadata.gz: 9c833575046c3a5436342f25caf5492296256cb8
+  data.tar.gz: 2e3a24748d0bc3a6030033a8a6d209f59aa40c24
 SHA512:
-  metadata.gz: 62b5bc7ece0fcef5d3c11f3c3305657c3862a63ea75e49fa57bc0b8dd0c4de0efbe24201124a74ab066f819776da9b35f434607b305b94175b31f80d0d4043cc
-  data.tar.gz: 9a032d7d46e1b2b365a0ca7536d8e8cd2b777a617c5274b99b59afcaae429da6fa9f0df8c02468aeb71189a38c1d671020f985d45df3a6c60332d6a4af92ce70
+  metadata.gz: 92bc8cbea288d93e02dbe9b9821c5cf47a8dea1484bd1d98094c8f45897ecdcff963393b7e555930a64701b2d01a5f850cede162951565ba077182d5646ac753
+  data.tar.gz: 0331daacc90ffd8ad3e998c8e465fce18f71b0f15550bbe08cbd102ae89a0d799ac979e9e2383da7e8cdec3708372220bacb001c0e440b53a92068f5654d851a

data/.travis.yml

@@ -1,15 +1,17 @@
 language: ruby
 rvm:
   - 1.8.7
-  - 1.9.2
+  - ree
   - 1.9.3
   - 2.0.0
-  - ree
+  - ruby-head
   - jruby-18mode
   - jruby-19mode
+  - jruby-head
   - rbx-18mode
   - rbx-19mode
 matrix:
   allow_failures:
+    - rvm: ruby-head
     - rvm: rbx-18mode
     - rvm: rbx-19mode

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2010, 2011, 2012, 2013 Akinori MUSHA
+Copyright (c) 2010-2016 Akinori MUSHA
 
 All rights reserved.
 

data/README.rdoc

@@ -34,5 +34,5 @@ This is a library to help write robots.txt compliant web robots.
 
 == Copyright
 
-Copyright (c) 2010, 2011, 2012, 2013 Akinori MUSHA. See LICENSE.txt for
+Copyright (c) 2010-2016 Akinori MUSHA. See LICENSE.txt for
 further details.

data/Rakefile

@@ -7,6 +7,7 @@ gemspec = Bundler::GemHelper.gemspec
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'test'
+  test.ruby_opts << '-r./test/simplecov_start.rb' unless RUBY_VERSION < '1.9' || (RUBY_PLATFORM == 'java' && ENV['TRAVIS'])
   test.test_files = gemspec.test_files
   test.verbose = true
 end

data/lib/webrobots.rb

@@ -171,8 +171,9 @@ class WebRobots
   end
 
   def http_get(uri)
+    response = nil
     referer = nil
-    10.times {
+    5.times {
       http = Net::HTTP.new(uri.host, uri.port)
       if http.use_ssl = uri.is_a?(URI::HTTPS)
         http.verify_mode = OpenSSL::SSL::VERIFY_PEER
@@ -190,13 +191,17 @@ class WebRobots
       when Net::HTTPRedirection
         referer = uri.to_s
         uri = URI(response['location'])
-      when Net::HTTPNotFound
+      when Net::HTTPClientError
         return ''
-      else
-        response.value
       end
     }
-    raise 'too many HTTP redirects'
+    case response
+    when Net::HTTPRedirection
+      # Treat too many redirections as not found
+      ''
+    else
+      raise "#{response.code} #{response.message}"
+    end
   end
 
   def crawl_delay_handler(delay, last_checked_at)

data/lib/webrobots/version.rb

@@ -1,3 +1,3 @@
 module Webrobots
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/test/helper.rb

@@ -9,6 +9,13 @@ rescue Bundler::BundlerError => e
 end
 require 'test/unit'
 require 'shoulda'
+require 'webmock/test_unit'
+require 'vcr'
+
+VCR.configure do |c|
+  c.cassette_library_dir = 'test/vcr_cassettes'
+  c.hook_into :webmock
+end
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))

data/test/simplecov_start.rb

@@ -0,0 +1,9 @@
+require 'simplecov'
+require 'coveralls'
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
+    SimpleCov::Formatter::HTMLFormatter,
+    Coveralls::SimpleCov::Formatter
+])
+
+SimpleCov.start

data/test/test_webrobots.rb

@@ -31,7 +31,7 @@ class TestWebRobots < Test::Unit::TestCase
         })
     end
 
-    should "allow any robot" do
+    should "be treated as full allow" do
       assert @robots.allowed?('http://site1.example.org/index.html')
       assert @robots.allowed?('http://site1.example.org/private/secret.txt')
       assert @robots.allowed?('http://site2.example.org/index.html')
@@ -43,7 +43,69 @@ class TestWebRobots < Test::Unit::TestCase
     end
   end
 
-  context "robots.txt that cannot be fetched" do
+  context "the default http_get proc" do
+    setup do
+      @robots = WebRobots.new('SomeBot')
+      @site_uri = URI('http://example.com/')
+      @txt_uri  = @site_uri + '/robots.txt'
+    end
+
+    should "treat 201 as success" do
+      stub_request(:get, @txt_uri).to_return(status: 201, body: '# Just created!')
+      assert_equal '# Just created!', @robots.__send__(:http_get, @txt_uri)
+      assert_requested :get, @txt_uri
+    end
+
+    should "treat redirects more than 5 times as full allow" do
+      redirect_uri = @site_uri + '/redirect'
+      stub_request(:get, /\A#{Regexp.quote(@site_uri.to_s)}/).to_return(status: 301, headers: { Location: redirect_uri.to_s })
+      assert_equal '', @robots.__send__(:http_get, @txt_uri)
+      assert_requested :get, @txt_uri, times: 1
+      assert_requested :get, redirect_uri, times: 4
+    end
+
+    should "treat 400 as full allow" do
+      stub_request(:get, @txt_uri).to_return(status: 400, body: 'Bad Request!')
+      assert_equal '', @robots.__send__(:http_get, @txt_uri)
+      assert_requested :get, @txt_uri
+    end
+
+    should "treat 401 as full allow" do
+      stub_request(:get, @txt_uri).to_return(status: 401, body: 'Unauthorized!')
+      assert_equal '', @robots.__send__(:http_get, @txt_uri)
+      assert_requested :get, @txt_uri
+    end
+
+    should "treat 403 as full allow" do
+      stub_request(:get, @txt_uri).to_return(status: 403, body: 'Forbidden!')
+      assert_equal '', @robots.__send__(:http_get, @txt_uri)
+      assert_requested :get, @txt_uri
+    end
+
+    should "treat 404 as full allow" do
+      stub_request(:get, @txt_uri).to_return(status: 404, body: 'Not Found!')
+      assert_equal '', @robots.__send__(:http_get, @txt_uri)
+      assert_requested :get, @txt_uri
+    end
+
+    should "treat 500 as error after retrying 5 times" do
+      stub_request(:get, @txt_uri).to_return(status: 500, body: 'Internal Server Error!')
+      assert_raise {
+        @robots.__send__(:http_get, @txt_uri)
+      }
+      assert_requested :get, @txt_uri, times: 5
+    end
+
+    should "treat 503 as error after retrying 5 times" do
+      stub_request(:get, @txt_uri).to_return(status: 503, body: 'Service Unavailable!')
+      assert_raise {
+        @robots.__send__(:http_get, @txt_uri)
+      }
+      assert_requested :get, @txt_uri, times: 5
+    end
+  end
+
+  context "nil or error from a custom http_get proc" do
     setup do
       @robots = WebRobots.new('RandomBot', :http_get => lambda { |uri|
           case uri.to_s
@@ -52,9 +114,8 @@ class TestWebRobots < Test::Unit::TestCase
               'Internal Server Error',
               Net::HTTPInternalServerError.new('1.1', '500', 'Internal Server Error'))
           when 'http://site2.example.org/robots.txt'
-            raise Net::HTTPRetriableError.new(
-              'Found',
-              Net::HTTPFound.new('1.1', '302', 'Found'))
+            require 'timeout'
+            Timeout.timeout(0) {}
           when 'http://site3.example.org/robots.txt'
             raise Errno::ECONNREFUSED
           when 'http://site4.example.org/robots.txt'
@@ -67,7 +128,7 @@ class TestWebRobots < Test::Unit::TestCase
         })
     end
 
-    should "disallow any robot" do
+    should "be treated as full disallow" do
       assert @robots.disallowed?('http://site1.example.org/index.html')
       assert @robots.disallowed?('http://site1.example.org/private/secret.txt')
       assert @robots.disallowed?('http://site2.example.org/index.html')
@@ -412,7 +473,7 @@ Option3: Hi
       @robots_hisbot = WebRobots.new('HisBot', :http_get => http_get)
     end
 
-    should "read options" do
+    should "be properly read" do
       options = @robots_mybot.options('http://www.example.org/')
       assert_equal 2, options.size
       assert_equal 'Foo',   @robots_mybot.option('http://www.example.org/', 'Option1')
@@ -531,28 +592,28 @@ Disallow: /
   context "robots.txt in the real world" do
     setup do
       @testbot = WebRobots.new('TestBot')
-      @msnbot = WebRobots.new('TestMSNBot')	# matches msnbot
+      @baidubot = WebRobots.new('TestBaiduspider')	# matches Baiduspider
     end
 
     should "be parsed for major sites" do
-      assert_nothing_raised {
-        assert !@testbot.allowed?("http://www.google.com/search")
-        assert !@testbot.allowed?("https://www.google.com/search")
-        assert !@testbot.allowed?("http://www.google.com/news/section?pz=1&cf=all&ned=jp&topic=y&ict=ln")
-        assert @testbot.allowed?("http://www.google.com/news/directory?pz=1&cf=all&ned=us&hl=en&sort=users&category=6")
-      }
-      assert_nothing_raised {
-        assert @testbot.allowed?("http://www.yahoo.com/")
-        assert !@testbot.allowed?("http://www.yahoo.com/?")
-        assert !@testbot.allowed?("http://www.yahoo.com/p/foo")
-      }
-      assert_nothing_raised {
-        assert !@testbot.allowed?("http://store.apple.com/vieworder")
-        assert @msnbot.allowed?("http://store.apple.com/vieworder")
-      }
-      assert_nothing_raised {
-        assert !@testbot.allowed?("http://github.com/login")
-      }
+      VCR.use_cassette("major_sites") do
+        assert_nothing_raised {
+          assert !@testbot.allowed?("https://www.google.com/search")
+          assert !@testbot.allowed?("https://www.google.com/catalogs")
+          assert @testbot.allowed?("https://www.google.com/catalogs/about")
+        }
+        assert_nothing_raised {
+          assert @testbot.allowed?("https://www.yahoo.com/")
+          assert !@testbot.allowed?("https://www.yahoo.com/p/foo")
+        }
+        assert_nothing_raised {
+          assert @testbot.allowed?("http://www.apple.com/jp/foo")
+          assert !@baidubot.allowed?("http://www.apple.com/jp/foo")
+        }
+        assert_nothing_raised {
+          assert !@testbot.allowed?("https://github.com/login")
+        }
+      end
     end
   end
 

data/test/vcr_cassettes/major_sites.yml

@@ -0,0 +1,2006 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://www.google.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBot
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Vary:
+      - Accept-Encoding
+      Content-Type:
+      - text/plain
+      Date:
+      - Mon, 04 Jan 2016 13:39:15 GMT
+      Expires:
+      - Mon, 04 Jan 2016 13:39:15 GMT
+      Cache-Control:
+      - private, max-age=0
+      Last-Modified:
+      - Thu, 17 Dec 2015 01:06:42 GMT
+      X-Content-Type-Options:
+      - nosniff
+      Server:
+      - sffe
+      X-Xss-Protection:
+      - 1; mode=block
+      Alternate-Protocol:
+      - 443:quic,p=1
+      Alt-Svc:
+      - quic="www.google.com:443"; ma=600; v="30,29,28,27,26,25",quic=":443"; ma=600;
+        v="30,29,28,27,26,25"
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: ASCII-8BIT
+      string: |+
+        User-agent: *
+        Disallow: /search
+        Allow: /search/about
+        Disallow: /sdch
+        Disallow: /groups
+        Disallow: /catalogs
+        Allow: /catalogs/about
+        Allow: /catalogs/p?
+        Disallow: /catalogues
+        Allow: /newsalerts
+        Disallow: /news
+        Allow: /news/directory
+        Disallow: /nwshp
+        Disallow: /setnewsprefs?
+        Disallow: /index.html?
+        Disallow: /?
+        Allow: /?hl=
+        Disallow: /?hl=*&
+        Allow: /?hl=*&gws_rd=ssl$
+        Disallow: /?hl=*&*&gws_rd=ssl
+        Allow: /?gws_rd=ssl$
+        Allow: /?pt1=true$
+        Disallow: /addurl/image?
+        Allow:    /mail/help/
+        Disallow: /mail/
+        Disallow: /pagead/
+        Disallow: /relpage/
+        Disallow: /relcontent
+        Disallow: /imgres
+        Disallow: /imglanding
+        Disallow: /sbd
+        Disallow: /keyword/
+        Disallow: /u/
+        Disallow: /univ/
+        Disallow: /cobrand
+        Disallow: /custom
+        Disallow: /advanced_group_search
+        Disallow: /googlesite
+        Disallow: /preferences
+        Disallow: /setprefs
+        Disallow: /swr
+        Disallow: /url
+        Disallow: /default
+        Disallow: /m?
+        Disallow: /m/
+        Allow:    /m/finance
+        Disallow: /wml?
+        Disallow: /wml/?
+        Disallow: /wml/search?
+        Disallow: /xhtml?
+        Disallow: /xhtml/?
+        Disallow: /xhtml/search?
+        Disallow: /xml?
+        Disallow: /imode?
+        Disallow: /imode/?
+        Disallow: /imode/search?
+        Disallow: /jsky?
+        Disallow: /jsky/?
+        Disallow: /jsky/search?
+        Disallow: /pda?
+        Disallow: /pda/?
+        Disallow: /pda/search?
+        Disallow: /sprint_xhtml
+        Disallow: /sprint_wml
+        Disallow: /pqa
+        Disallow: /palm
+        Disallow: /gwt/
+        Disallow: /purchases
+        Disallow: /bsd?
+        Disallow: /linux?
+        Disallow: /mac?
+        Disallow: /microsoft?
+        Disallow: /unclesam?
+        Disallow: /answers/search?q=
+        Disallow: /local?
+        Disallow: /local_url
+        Disallow: /shihui?
+        Disallow: /shihui/
+        Disallow: /froogle?
+        Disallow: /products?
+        Disallow: /froogle_
+        Disallow: /product_
+        Disallow: /products_
+        Disallow: /products;
+        Disallow: /print
+        Disallow: /books/
+        Disallow: /bkshp?*q=*
+        Disallow: /books?*q=*
+        Disallow: /books?*output=*
+        Disallow: /books?*pg=*
+        Disallow: /books?*jtp=*
+        Disallow: /books?*jscmd=*
+        Disallow: /books?*buy=*
+        Disallow: /books?*zoom=*
+        Allow: /books?*q=related:*
+        Allow: /books?*q=editions:*
+        Allow: /books?*q=subject:*
+        Allow: /books/about
+        Allow: /booksrightsholders
+        Allow: /books?*zoom=1*
+        Allow: /books?*zoom=5*
+        Disallow: /ebooks/
+        Disallow: /ebooks?*q=*
+        Disallow: /ebooks?*output=*
+        Disallow: /ebooks?*pg=*
+        Disallow: /ebooks?*jscmd=*
+        Disallow: /ebooks?*buy=*
+        Disallow: /ebooks?*zoom=*
+        Allow: /ebooks?*q=related:*
+        Allow: /ebooks?*q=editions:*
+        Allow: /ebooks?*q=subject:*
+        Allow: /ebooks?*zoom=1*
+        Allow: /ebooks?*zoom=5*
+        Disallow: /patents?
+        Disallow: /patents/download/
+        Disallow: /patents/pdf/
+        Disallow: /patents/related/
+        Disallow: /scholar
+        Disallow: /citations?
+        Allow: /citations?user=
+        Disallow: /citations?*cstart=
+        Allow: /citations?view_op=new_profile
+        Allow: /citations?view_op=top_venues
+        Disallow: /complete
+        Disallow: /s?
+        Disallow: /sponsoredlinks
+        Disallow: /videosearch?
+        Disallow: /videopreview?
+        Disallow: /videoprograminfo?
+        Allow: /maps?*output=classic*
+        Allow: /maps?*file=
+        Allow: /maps/api/js?
+        Allow: /maps/d/
+        Disallow: /maps?
+        Disallow: /mapstt?
+        Disallow: /mapslt?
+        Disallow: /maps/stk/
+        Disallow: /maps/br?
+        Disallow: /mapabcpoi?
+        Disallow: /maphp?
+        Disallow: /mapprint?
+        Disallow: /maps/api/js/
+        Disallow: /maps/api/staticmap?
+        Disallow: /maps/api/streetview
+        Disallow: /mld?
+        Disallow: /staticmap?
+        Disallow: /places/
+        Allow: /places/$
+        Disallow: /maps/preview
+        Disallow: /maps/place
+        Disallow: /help/maps/streetview/partners/welcome/
+        Disallow: /help/maps/indoormaps/partners/
+        Disallow: /lochp?
+        Disallow: /center
+        Disallow: /ie?
+        Disallow: /sms/demo?
+        Disallow: /katrina?
+        Disallow: /blogsearch?
+        Disallow: /blogsearch/
+        Disallow: /blogsearch_feeds
+        Disallow: /advanced_blog_search
+        Disallow: /uds/
+        Disallow: /chart?
+        Disallow: /transit?
+        Disallow: /mbd?
+        Disallow: /extern_js/
+        Disallow: /xjs/
+        Disallow: /calendar/feeds/
+        Disallow: /calendar/ical/
+        Disallow: /cl2/feeds/
+        Disallow: /cl2/ical/
+        Disallow: /coop/directory
+        Disallow: /coop/manage
+        Disallow: /trends?
+        Disallow: /trends/music?
+        Disallow: /trends/hottrends?
+        Disallow: /trends/viz?
+        Disallow: /trends/embed.js?
+        Disallow: /trends/fetchComponent?
+        Disallow: /notebook/search?
+        Disallow: /musica
+        Disallow: /musicad
+        Disallow: /musicas
+        Disallow: /musicl
+        Disallow: /musics
+        Disallow: /musicsearch
+        Disallow: /musicsp
+        Disallow: /musiclp
+        Disallow: /browsersync
+        Disallow: /call
+        Disallow: /archivesearch?
+        Disallow: /archivesearch/url
+        Disallow: /archivesearch/advanced_search
+        Disallow: /base/reportbadoffer
+        Disallow: /urchin_test/
+        Disallow: /movies?
+        Disallow: /codesearch?
+        Disallow: /codesearch/feeds/search?
+        Disallow: /wapsearch?
+        Disallow: /safebrowsing
+        Allow: /safebrowsing/diagnostic
+        Allow: /safebrowsing/report_badware/
+        Allow: /safebrowsing/report_error/
+        Allow: /safebrowsing/report_phish/
+        Disallow: /reviews/search?
+        Disallow: /orkut/albums
+        Allow: /jsapi
+        Disallow: /views?
+        Disallow: /c/
+        Disallow: /cbk
+        Allow: /cbk?output=tile&cb_client=maps_sv
+        Disallow: /kh
+        Disallow: /vt
+        Disallow: /maps/vt
+        Disallow: /recharge/dashboard/car
+        Disallow: /recharge/dashboard/static/
+        Disallow: /translate_a/
+        Disallow: /translate_c
+        Disallow: /translate_f
+        Disallow: /translate_static/
+        Disallow: /translate_suggestion
+        Disallow: /profiles/me
+        Allow: /profiles
+        Disallow: /s2/profiles/me
+        Allow: /s2/profiles
+        Allow: /s2/oz
+        Allow: /s2/photos
+        Allow: /s2/search/social
+        Allow: /s2/static
+        Disallow: /s2
+        Disallow: /transconsole/portal/
+        Disallow: /gcc/
+        Disallow: /aclk
+        Disallow: /cse?
+        Disallow: /cse/home
+        Disallow: /cse/panel
+        Disallow: /cse/manage
+        Disallow: /tbproxy/
+        Disallow: /imesync/
+        Disallow: /shenghuo/search?
+        Disallow: /support/forum/search?
+        Disallow: /reviews/polls/
+        Disallow: /hosted/images/
+        Disallow: /ppob/?
+        Disallow: /ppob?
+        Disallow: /adwordsresellers
+        Disallow: /accounts/ClientLogin
+        Disallow: /accounts/ClientAuth
+        Disallow: /accounts/o8
+        Allow: /accounts/o8/id
+        Disallow: /topicsearch?q=
+        Disallow: /xfx7/
+        Disallow: /squared/api
+        Disallow: /squared/search
+        Disallow: /squared/table
+        Disallow: /toolkit/
+        Allow: /toolkit/*.html
+        Disallow: /globalmarketfinder/
+        Allow: /globalmarketfinder/*.html
+        Disallow: /qnasearch?
+        Disallow: /app/updates
+        Disallow: /sidewiki/entry/
+        Disallow: /quality_form?
+        Disallow: /labs/popgadget/search
+        Disallow: /buzz/post
+        Disallow: /compressiontest/
+        Disallow: /analytics/reporting/
+        Disallow: /analytics/admin/
+        Disallow: /analytics/web/
+        Disallow: /analytics/feeds/
+        Disallow: /analytics/settings/
+        Allow: /alerts/manage
+        Allow: /alerts/remove
+        Disallow: /alerts/
+        Allow: /alerts/$
+        Disallow: /ads/search?
+        Disallow: /ads/plan/action_plan?
+        Disallow: /ads/plan/api/
+        Disallow: /ads/hotels/partners
+        Disallow: /phone/compare/?
+        Disallow: /travel/clk
+        Disallow: /hotelfinder/rpc
+        Disallow: /hotels/rpc
+        Disallow: /flights/rpc
+        Disallow: /commercesearch/services/
+        Disallow: /evaluation/
+        Disallow: /chrome/browser/mobile/tour
+        Disallow: /compare/*/apply*
+        Disallow: /forms/perks/
+        Disallow: /baraza/*/search
+        Disallow: /baraza/*/report
+        Disallow: /shopping/suppliers/search
+        Disallow: /ct/
+        Disallow: /edu/cs4hs/
+        Disallow: /trustedstores/s/
+        Disallow: /trustedstores/tm2
+        Disallow: /trustedstores/verify
+        Disallow: /adwords/proposal
+        Disallow: /shopping/product/
+        Disallow: /shopping/seller
+        Disallow: /shopping/reviewer
+        Disallow: /about/careers/apply/
+        Disallow: /about/careers/applications/
+        Disallow: /landing/signout.html
+        Disallow: /webmasters/sitemaps/ping?
+        Disallow: /ping?
+        Allow: /gb/images
+        Allow: /gb/js
+        Disallow: /gallery/
+        Allow: /chromecast/setup$
+        Allow: /chromecast/setup/$
+        Disallow: /landing/now/ontap/
+        Sitemap: http://www.gstatic.com/culturalinstitute/sitemaps/www_google_com_culturalinstitute/sitemap-index.xml
+        Sitemap: http://www.gstatic.com/dictionary/static/sitemaps/sitemap_index.xml
+        Sitemap: http://www.gstatic.com/earth/gallery/sitemaps/sitemap.xml
+        Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml
+        Sitemap: http://www.gstatic.com/trends/websites/sitemaps/sitemapindex.xml
+        Sitemap: https://www.google.com/sitemap.xml
+
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:15 GMT
+- request:
+    method: get
+    uri: https://www.yahoo.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBot
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 04 Jan 2016 13:39:16 GMT
+      P3p:
+      - policyref="http://info.yahoo.com/w3c/p3p.xml", CP="CAO DSP COR CUR ADM DEV
+        TAI PSA PSD IVAi IVDi CONi TELo OTPi OUR DELi SAMi OTRi UNRi PUBi IND PHY
+        ONL UNI PUR FIN COM NAV INT DEM CNT STA POL HEA PRE LOC GOV"
+      Cache-Control:
+      - private
+      Last-Modified:
+      - Thu, 19 Nov 2015 00:31:02 GMT
+      Accept-Ranges:
+      - bytes
+      Vary:
+      - Accept-Encoding
+      Content-Type:
+      - text/plain; charset=utf-8
+      Age:
+      - '2'
+      Via:
+      - http/1.1 usproxy2.fp.gq1.yahoo.com (ApacheTrafficServer), http/1.1 ir2.fp.sg3.yahoo.com
+        (ApacheTrafficServer)
+      Server:
+      - ATS
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Y-Trace:
+      - BAEAQAAAAAADb4_wm_pNEAAAAAAAAAAAGXeCK6w0SUwAAAAAAAAAAAAFKII_Uu55AAUogj9WSEhTfIBlAAAAAA--
+    body:
+      encoding: ASCII-8BIT
+      string: |
+        User-agent: *
+        Disallow: /p/
+        Disallow: /r/
+        Disallow: /bin/
+        Disallow: /includes/
+        Disallow: /blank.html
+        Disallow: /_td_api
+        Disallow: /_tdpp_api
+        Disallow: /_remote
+        Disallow: /_multiremote
+        Disallow: /_tdhl_api
+
+        Sitemap: https://www.yahoo.com/food/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/tech/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/travel/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/movies/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/beauty/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/health/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/style/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/makers/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/parenting/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/music/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/tv/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/politics/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/autos/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/katiecouric/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/digest/sitemap.xml
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:17 GMT
+- request:
+    method: get
+    uri: http://www.apple.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBot
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Last-Modified:
+      - Thu, 22 Oct 2015 00:20:47 GMT
+      Server:
+      - Apache
+      Nncoection:
+      - close
+      Accept-Ranges:
+      - bytes
+      Content-Length:
+      - '6902'
+      Content-Type:
+      - text/plain; charset=UTF-8
+      Cache-Control:
+      - max-age=561
+      Expires:
+      - Mon, 04 Jan 2016 13:48:38 GMT
+      Date:
+      - Mon, 04 Jan 2016 13:39:17 GMT
+      Connection:
+      - keep-alive
+    body:
+      encoding: UTF-8
+      string: "# robots.txt for http://www.apple.com/\n\nUser-agent: *\nDisallow:
+        /*/includes/*\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/answer/vote*\nDisallow: /*/shop/bag*\nDisallow:
+        /*/shop/browse/campaigns/mobile_overlay*\nDisallow: /*/shop/button_availability*\nDisallow:
+        /*/shop/favorites*\nDisallow: /*/shop/mobile/olss_error*\nDisallow: /*/shop/mobilex/*\nDisallow:
+        /*/shop/order/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow:
+        /*/shop/question/unsubscribe/\nDisallow: /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow:
+        /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/store/feeds/*\nDisallow: /*/shop/variationSelection\nDisallow: /*_adc_*/shop/\nDisallow:
+        /*_aoc_*/shop/\nDisallow: /*_edu_*/shop/\nDisallow: /*_enterprise*/shop/\nDisallow:
+        /*_internal-epp-discounted*/shop/\nDisallow: /*_k12nonbts*/shop/\nDisallow:
+        /*_kiosk*/shop/\nDisallow: /*_nonbts*/shop/\nDisallow: /*_qpromo*/shop/\nDisallow:
+        /*_refurb-discounted*/shop/\nDisallow: /cn/se/*\nDisallow: /retail/availability*\nDisallow:
+        /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow:
+        /search/*\nDisallow: /shop/*signed_in_account*\nDisallow: /shop/*sign_in*\nDisallow:
+        /shop/*sign_out*\nDisallow: /shop/*WebObjects/*\nDisallow: /shop/1-800-MY-APPLE/*\nDisallow:
+        /shop/answer/vote*\nDisallow: /shop/bag*\nDisallow: /shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /shop/button_availability*\nDisallow: /shop/favorites*\nDisallow: /shop/mobile/olss_error*\nDisallow:
+        /shop/mobilex/\nDisallow: /shop/order/*\nDisallow: /shop/question/answer/report*\nDisallow:
+        /shop/question/subscribe*\nDisallow: /shop/question/unsubscribe/\nDisallow:
+        /shop/review/vote*\nDisallow: /shop/reviews/report*\nDisallow: /shop/rs-mvt/rel/*\nDisallow:
+        /shop/search/*\nDisallow: /shop/sentry*\nDisallow: /shop/socialsharing/*\nDisallow:
+        /shop/store/feeds/*\nDisallow: /shop/variationSelection*\nDisallow: /tmall*\nAllow:
+        /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent:
+        Baiduspider\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow:
+        /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /*/shop/browse/overlay*\nDisallow: /*/shop/button_availability*\nDisallow:
+        /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow: /*/shop/checkout*\nDisallow:
+        /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow: /*/shop/identify_user*\nDisallow:
+        /*/shop/mobile/checkout/start*\nDisallow: /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow:
+        /*/shop/question/subscribe*\nDisallow: /*/shop/question/unsubscribe/*\nDisallow:
+        /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow:
+        /*/shop/sentry*\nDisallow: /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow:
+        /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow:
+        /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/storeConfig*\nDisallow: /*/shop/vieworder*\nDisallow: /*/variationSelection*\nDisallow:
+        /ae/*\nDisallow: /am/*\nDisallow: /asia/*\nDisallow: /at/*\nDisallow: /au/*\nDisallow:
+        /befr/*\nDisallow: /benl/*\nDisallow: /bg/*\nDisallow: /bh/*\nDisallow: /br/*\nDisallow:
+        /bw/*\nDisallow: /ca/*\nDisallow: /ca/fr/*\nDisallow: /cf/*\nDisallow: /chde/*\nDisallow:
+        /chfr/*\nDisallow: /ci/*\nDisallow: /cl/*\nDisallow: /cm/*\nDisallow: /cn/se/*\nDisallow:
+        /co/*\nDisallow: /cz/*\nDisallow: /de/*\nDisallow: /dk/*\nDisallow: /ee/*\nDisallow:
+        /eg/*\nDisallow: /es/*\nDisallow: /fi/*\nDisallow: /fr/*\nDisallow: /gn/*\nDisallow:
+        /gq/*\nDisallow: /gr/*\nDisallow: /gw/*\nDisallow: /hk/*\nDisallow: /hk/en/*\nDisallow:
+        /hr/*\nDisallow: /hu/*\nDisallow: /id/*\nDisallow: /ie/*\nDisallow: /il/*\nDisallow:
+        /in/*\nDisallow: /it/*\nDisallow: /jo/*\nDisallow: /jp/*\nDisallow: /ke/*\nDisallow:
+        /kr/*\nDisallow: /kw/*\nDisallow: /la/*\nDisallow: /lae/*\nDisallow: /li/*\nDisallow:
+        /lt/*\nDisallow: /lu/*\nDisallow: /lv/*\nDisallow: /ma/*\nDisallow: /md/*\nDisallow:
+        /me/*\nDisallow: /mg/*\nDisallow: /mk/*\nDisallow: /ml/*\nDisallow: /mt/*\nDisallow:
+        /mu/*\nDisallow: /mx/*\nDisallow: /my/*\nDisallow: /mz/*\nDisallow: /ne/*\nDisallow:
+        /ng/*\nDisallow: /nl/*\nDisallow: /no/*\nDisallow: /nz/*\nDisallow: /om/*\nDisallow:
+        /ph/*\nDisallow: /pl/*\nDisallow: /pt/*\nDisallow: /qa/*\nDisallow: /retail/availability*\nDisallow:
+        /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow:
+        /ro/*\nDisallow: /ru/*\nDisallow: /sa/*\nDisallow: /se/*\nDisallow: /search/*\nDisallow:
+        /sg/*\nDisallow: /shop/*\nDisallow: /si/*\nDisallow: /sk/*\nDisallow: /sn/*\nDisallow:
+        /th/*\nDisallow: /tmall/*\nDisallow: /tn/*\nDisallow: /tr/*\nDisallow: /tw/*\nDisallow:
+        /ug/*\nDisallow: /uk/*\nDisallow: /vn/*\nDisallow: /za/*\n\nUser-agent: HaoSouSpider\nDisallow:
+        /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow:
+        /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /*/shop/button_availability*\nDisallow: /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow:
+        /*/shop/checkout*\nDisallow: /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow:
+        /*/shop/identify_user*\nDisallow: /*/shop/mobile/checkout/start*\nDisallow:
+        /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow:
+        /*/shop/question/unsubscribe/*\nDisallow: /*/shop/review/vote*\nDisallow:
+        /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow:
+        /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow:
+        /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow:
+        /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/storeConfig*\nDisallow: /*/shop/variationSelection*\nDisallow: /*/shop/vieworder*\nDisallow:
+        /cn/se/*\nDisallow: /retail/availability*\nDisallow: /retail/availabilitySearch*\nDisallow:
+        /retail/pickupEligibility*\nDisallow: /search/*\nDisallow: /shop/bag*\nDisallow:
+        /shop/search/*\nDisallow: /tmall/*\n\nSitemap: http://www.apple.com/shop/sitemap.xml\nSitemap:
+        http://www.apple.com/sitemap.xml\n"
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:17 GMT
+- request:
+    method: get
+    uri: http://www.apple.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBaiduspider
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Last-Modified:
+      - Thu, 22 Oct 2015 00:20:47 GMT
+      Server:
+      - Apache
+      Nncoection:
+      - close
+      Accept-Ranges:
+      - bytes
+      Content-Length:
+      - '6902'
+      Content-Type:
+      - text/plain; charset=UTF-8
+      Cache-Control:
+      - max-age=417
+      Expires:
+      - Mon, 04 Jan 2016 13:46:14 GMT
+      Date:
+      - Mon, 04 Jan 2016 13:39:17 GMT
+      Connection:
+      - keep-alive
+    body:
+      encoding: UTF-8
+      string: "# robots.txt for http://www.apple.com/\n\nUser-agent: *\nDisallow:
+        /*/includes/*\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/answer/vote*\nDisallow: /*/shop/bag*\nDisallow:
+        /*/shop/browse/campaigns/mobile_overlay*\nDisallow: /*/shop/button_availability*\nDisallow:
+        /*/shop/favorites*\nDisallow: /*/shop/mobile/olss_error*\nDisallow: /*/shop/mobilex/*\nDisallow:
+        /*/shop/order/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow:
+        /*/shop/question/unsubscribe/\nDisallow: /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow:
+        /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/store/feeds/*\nDisallow: /*/shop/variationSelection\nDisallow: /*_adc_*/shop/\nDisallow:
+        /*_aoc_*/shop/\nDisallow: /*_edu_*/shop/\nDisallow: /*_enterprise*/shop/\nDisallow:
+        /*_internal-epp-discounted*/shop/\nDisallow: /*_k12nonbts*/shop/\nDisallow:
+        /*_kiosk*/shop/\nDisallow: /*_nonbts*/shop/\nDisallow: /*_qpromo*/shop/\nDisallow:
+        /*_refurb-discounted*/shop/\nDisallow: /cn/se/*\nDisallow: /retail/availability*\nDisallow:
+        /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow:
+        /search/*\nDisallow: /shop/*signed_in_account*\nDisallow: /shop/*sign_in*\nDisallow:
+        /shop/*sign_out*\nDisallow: /shop/*WebObjects/*\nDisallow: /shop/1-800-MY-APPLE/*\nDisallow:
+        /shop/answer/vote*\nDisallow: /shop/bag*\nDisallow: /shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /shop/button_availability*\nDisallow: /shop/favorites*\nDisallow: /shop/mobile/olss_error*\nDisallow:
+        /shop/mobilex/\nDisallow: /shop/order/*\nDisallow: /shop/question/answer/report*\nDisallow:
+        /shop/question/subscribe*\nDisallow: /shop/question/unsubscribe/\nDisallow:
+        /shop/review/vote*\nDisallow: /shop/reviews/report*\nDisallow: /shop/rs-mvt/rel/*\nDisallow:
+        /shop/search/*\nDisallow: /shop/sentry*\nDisallow: /shop/socialsharing/*\nDisallow:
+        /shop/store/feeds/*\nDisallow: /shop/variationSelection*\nDisallow: /tmall*\nAllow:
+        /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent:
+        Baiduspider\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow:
+        /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /*/shop/browse/overlay*\nDisallow: /*/shop/button_availability*\nDisallow:
+        /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow: /*/shop/checkout*\nDisallow:
+        /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow: /*/shop/identify_user*\nDisallow:
+        /*/shop/mobile/checkout/start*\nDisallow: /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow:
+        /*/shop/question/subscribe*\nDisallow: /*/shop/question/unsubscribe/*\nDisallow:
+        /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow:
+        /*/shop/sentry*\nDisallow: /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow:
+        /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow:
+        /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/storeConfig*\nDisallow: /*/shop/vieworder*\nDisallow: /*/variationSelection*\nDisallow:
+        /ae/*\nDisallow: /am/*\nDisallow: /asia/*\nDisallow: /at/*\nDisallow: /au/*\nDisallow:
+        /befr/*\nDisallow: /benl/*\nDisallow: /bg/*\nDisallow: /bh/*\nDisallow: /br/*\nDisallow:
+        /bw/*\nDisallow: /ca/*\nDisallow: /ca/fr/*\nDisallow: /cf/*\nDisallow: /chde/*\nDisallow:
+        /chfr/*\nDisallow: /ci/*\nDisallow: /cl/*\nDisallow: /cm/*\nDisallow: /cn/se/*\nDisallow:
+        /co/*\nDisallow: /cz/*\nDisallow: /de/*\nDisallow: /dk/*\nDisallow: /ee/*\nDisallow:
+        /eg/*\nDisallow: /es/*\nDisallow: /fi/*\nDisallow: /fr/*\nDisallow: /gn/*\nDisallow:
+        /gq/*\nDisallow: /gr/*\nDisallow: /gw/*\nDisallow: /hk/*\nDisallow: /hk/en/*\nDisallow:
+        /hr/*\nDisallow: /hu/*\nDisallow: /id/*\nDisallow: /ie/*\nDisallow: /il/*\nDisallow:
+        /in/*\nDisallow: /it/*\nDisallow: /jo/*\nDisallow: /jp/*\nDisallow: /ke/*\nDisallow:
+        /kr/*\nDisallow: /kw/*\nDisallow: /la/*\nDisallow: /lae/*\nDisallow: /li/*\nDisallow:
+        /lt/*\nDisallow: /lu/*\nDisallow: /lv/*\nDisallow: /ma/*\nDisallow: /md/*\nDisallow:
+        /me/*\nDisallow: /mg/*\nDisallow: /mk/*\nDisallow: /ml/*\nDisallow: /mt/*\nDisallow:
+        /mu/*\nDisallow: /mx/*\nDisallow: /my/*\nDisallow: /mz/*\nDisallow: /ne/*\nDisallow:
+        /ng/*\nDisallow: /nl/*\nDisallow: /no/*\nDisallow: /nz/*\nDisallow: /om/*\nDisallow:
+        /ph/*\nDisallow: /pl/*\nDisallow: /pt/*\nDisallow: /qa/*\nDisallow: /retail/availability*\nDisallow:
+        /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow:
+        /ro/*\nDisallow: /ru/*\nDisallow: /sa/*\nDisallow: /se/*\nDisallow: /search/*\nDisallow:
+        /sg/*\nDisallow: /shop/*\nDisallow: /si/*\nDisallow: /sk/*\nDisallow: /sn/*\nDisallow:
+        /th/*\nDisallow: /tmall/*\nDisallow: /tn/*\nDisallow: /tr/*\nDisallow: /tw/*\nDisallow:
+        /ug/*\nDisallow: /uk/*\nDisallow: /vn/*\nDisallow: /za/*\n\nUser-agent: HaoSouSpider\nDisallow:
+        /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow:
+        /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /*/shop/button_availability*\nDisallow: /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow:
+        /*/shop/checkout*\nDisallow: /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow:
+        /*/shop/identify_user*\nDisallow: /*/shop/mobile/checkout/start*\nDisallow:
+        /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow:
+        /*/shop/question/unsubscribe/*\nDisallow: /*/shop/review/vote*\nDisallow:
+        /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow:
+        /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow:
+        /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow:
+        /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/storeConfig*\nDisallow: /*/shop/variationSelection*\nDisallow: /*/shop/vieworder*\nDisallow:
+        /cn/se/*\nDisallow: /retail/availability*\nDisallow: /retail/availabilitySearch*\nDisallow:
+        /retail/pickupEligibility*\nDisallow: /search/*\nDisallow: /shop/bag*\nDisallow:
+        /shop/search/*\nDisallow: /tmall/*\n\nSitemap: http://www.apple.com/shop/sitemap.xml\nSitemap:
+        http://www.apple.com/sitemap.xml\n"
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:17 GMT
+- request:
+    method: get
+    uri: https://github.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBot
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - GitHub.com
+      Date:
+      - Mon, 04 Jan 2016 13:39:18 GMT
+      Content-Type:
+      - text/plain
+      Last-Modified:
+      - Mon, 04 Jan 2016 13:28:38 GMT
+      Transfer-Encoding:
+      - chunked
+      Etag:
+      - W/"568a7386-6e36"
+      Vary:
+      - Accept-Encoding
+      X-Served-By:
+      - a568c03544f42dddf712bab3bfd562fd
+      X-Github-Request-Id:
+      - 7C27244A:448A:3EB799EF:568A7605
+    body:
+      encoding: ASCII-8BIT
+      string: |
+        # If you would like to crawl GitHub contact us at support@github.com.
+        # We also provide an extensive API: https://developer.github.com/
+
+        User-agent: CCBot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: coccoc
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: dotbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: duckduckbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: EtaoSpider
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Googlebot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: HTTrack
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: ia_archiver
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: IntuitGSACrawler
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Mail.RU_Bot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: msnbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Bingbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: naverbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: red-app-gsa-p-one
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: rogerbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: SandDollar
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: seznambot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Slurp
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Swiftbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Telefonica
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: teoma
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Twitterbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Yandex
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+
+        User-agent: *
+        Allow: /humans.txt
+        Disallow: /
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:18 GMT
+recorded_with: VCR 3.0.0