husc 0.1.0

data/vendor/bundle/gems/webrobots-0.1.2/test/vcr_cassettes/major_sites.yml

@@ -0,0 +1,2006 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://www.google.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBot
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Vary:
+      - Accept-Encoding
+      Content-Type:
+      - text/plain
+      Date:
+      - Mon, 04 Jan 2016 13:39:15 GMT
+      Expires:
+      - Mon, 04 Jan 2016 13:39:15 GMT
+      Cache-Control:
+      - private, max-age=0
+      Last-Modified:
+      - Thu, 17 Dec 2015 01:06:42 GMT
+      X-Content-Type-Options:
+      - nosniff
+      Server:
+      - sffe
+      X-Xss-Protection:
+      - 1; mode=block
+      Alternate-Protocol:
+      - 443:quic,p=1
+      Alt-Svc:
+      - quic="www.google.com:443"; ma=600; v="30,29,28,27,26,25",quic=":443"; ma=600;
+        v="30,29,28,27,26,25"
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: ASCII-8BIT
+      string: |+
+        User-agent: *
+        Disallow: /search
+        Allow: /search/about
+        Disallow: /sdch
+        Disallow: /groups
+        Disallow: /catalogs
+        Allow: /catalogs/about
+        Allow: /catalogs/p?
+        Disallow: /catalogues
+        Allow: /newsalerts
+        Disallow: /news
+        Allow: /news/directory
+        Disallow: /nwshp
+        Disallow: /setnewsprefs?
+        Disallow: /index.html?
+        Disallow: /?
+        Allow: /?hl=
+        Disallow: /?hl=*&
+        Allow: /?hl=*&gws_rd=ssl$
+        Disallow: /?hl=*&*&gws_rd=ssl
+        Allow: /?gws_rd=ssl$
+        Allow: /?pt1=true$
+        Disallow: /addurl/image?
+        Allow:    /mail/help/
+        Disallow: /mail/
+        Disallow: /pagead/
+        Disallow: /relpage/
+        Disallow: /relcontent
+        Disallow: /imgres
+        Disallow: /imglanding
+        Disallow: /sbd
+        Disallow: /keyword/
+        Disallow: /u/
+        Disallow: /univ/
+        Disallow: /cobrand
+        Disallow: /custom
+        Disallow: /advanced_group_search
+        Disallow: /googlesite
+        Disallow: /preferences
+        Disallow: /setprefs
+        Disallow: /swr
+        Disallow: /url
+        Disallow: /default
+        Disallow: /m?
+        Disallow: /m/
+        Allow:    /m/finance
+        Disallow: /wml?
+        Disallow: /wml/?
+        Disallow: /wml/search?
+        Disallow: /xhtml?
+        Disallow: /xhtml/?
+        Disallow: /xhtml/search?
+        Disallow: /xml?
+        Disallow: /imode?
+        Disallow: /imode/?
+        Disallow: /imode/search?
+        Disallow: /jsky?
+        Disallow: /jsky/?
+        Disallow: /jsky/search?
+        Disallow: /pda?
+        Disallow: /pda/?
+        Disallow: /pda/search?
+        Disallow: /sprint_xhtml
+        Disallow: /sprint_wml
+        Disallow: /pqa
+        Disallow: /palm
+        Disallow: /gwt/
+        Disallow: /purchases
+        Disallow: /bsd?
+        Disallow: /linux?
+        Disallow: /mac?
+        Disallow: /microsoft?
+        Disallow: /unclesam?
+        Disallow: /answers/search?q=
+        Disallow: /local?
+        Disallow: /local_url
+        Disallow: /shihui?
+        Disallow: /shihui/
+        Disallow: /froogle?
+        Disallow: /products?
+        Disallow: /froogle_
+        Disallow: /product_
+        Disallow: /products_
+        Disallow: /products;
+        Disallow: /print
+        Disallow: /books/
+        Disallow: /bkshp?*q=*
+        Disallow: /books?*q=*
+        Disallow: /books?*output=*
+        Disallow: /books?*pg=*
+        Disallow: /books?*jtp=*
+        Disallow: /books?*jscmd=*
+        Disallow: /books?*buy=*
+        Disallow: /books?*zoom=*
+        Allow: /books?*q=related:*
+        Allow: /books?*q=editions:*
+        Allow: /books?*q=subject:*
+        Allow: /books/about
+        Allow: /booksrightsholders
+        Allow: /books?*zoom=1*
+        Allow: /books?*zoom=5*
+        Disallow: /ebooks/
+        Disallow: /ebooks?*q=*
+        Disallow: /ebooks?*output=*
+        Disallow: /ebooks?*pg=*
+        Disallow: /ebooks?*jscmd=*
+        Disallow: /ebooks?*buy=*
+        Disallow: /ebooks?*zoom=*
+        Allow: /ebooks?*q=related:*
+        Allow: /ebooks?*q=editions:*
+        Allow: /ebooks?*q=subject:*
+        Allow: /ebooks?*zoom=1*
+        Allow: /ebooks?*zoom=5*
+        Disallow: /patents?
+        Disallow: /patents/download/
+        Disallow: /patents/pdf/
+        Disallow: /patents/related/
+        Disallow: /scholar
+        Disallow: /citations?
+        Allow: /citations?user=
+        Disallow: /citations?*cstart=
+        Allow: /citations?view_op=new_profile
+        Allow: /citations?view_op=top_venues
+        Disallow: /complete
+        Disallow: /s?
+        Disallow: /sponsoredlinks
+        Disallow: /videosearch?
+        Disallow: /videopreview?
+        Disallow: /videoprograminfo?
+        Allow: /maps?*output=classic*
+        Allow: /maps?*file=
+        Allow: /maps/api/js?
+        Allow: /maps/d/
+        Disallow: /maps?
+        Disallow: /mapstt?
+        Disallow: /mapslt?
+        Disallow: /maps/stk/
+        Disallow: /maps/br?
+        Disallow: /mapabcpoi?
+        Disallow: /maphp?
+        Disallow: /mapprint?
+        Disallow: /maps/api/js/
+        Disallow: /maps/api/staticmap?
+        Disallow: /maps/api/streetview
+        Disallow: /mld?
+        Disallow: /staticmap?
+        Disallow: /places/
+        Allow: /places/$
+        Disallow: /maps/preview
+        Disallow: /maps/place
+        Disallow: /help/maps/streetview/partners/welcome/
+        Disallow: /help/maps/indoormaps/partners/
+        Disallow: /lochp?
+        Disallow: /center
+        Disallow: /ie?
+        Disallow: /sms/demo?
+        Disallow: /katrina?
+        Disallow: /blogsearch?
+        Disallow: /blogsearch/
+        Disallow: /blogsearch_feeds
+        Disallow: /advanced_blog_search
+        Disallow: /uds/
+        Disallow: /chart?
+        Disallow: /transit?
+        Disallow: /mbd?
+        Disallow: /extern_js/
+        Disallow: /xjs/
+        Disallow: /calendar/feeds/
+        Disallow: /calendar/ical/
+        Disallow: /cl2/feeds/
+        Disallow: /cl2/ical/
+        Disallow: /coop/directory
+        Disallow: /coop/manage
+        Disallow: /trends?
+        Disallow: /trends/music?
+        Disallow: /trends/hottrends?
+        Disallow: /trends/viz?
+        Disallow: /trends/embed.js?
+        Disallow: /trends/fetchComponent?
+        Disallow: /notebook/search?
+        Disallow: /musica
+        Disallow: /musicad
+        Disallow: /musicas
+        Disallow: /musicl
+        Disallow: /musics
+        Disallow: /musicsearch
+        Disallow: /musicsp
+        Disallow: /musiclp
+        Disallow: /browsersync
+        Disallow: /call
+        Disallow: /archivesearch?
+        Disallow: /archivesearch/url
+        Disallow: /archivesearch/advanced_search
+        Disallow: /base/reportbadoffer
+        Disallow: /urchin_test/
+        Disallow: /movies?
+        Disallow: /codesearch?
+        Disallow: /codesearch/feeds/search?
+        Disallow: /wapsearch?
+        Disallow: /safebrowsing
+        Allow: /safebrowsing/diagnostic
+        Allow: /safebrowsing/report_badware/
+        Allow: /safebrowsing/report_error/
+        Allow: /safebrowsing/report_phish/
+        Disallow: /reviews/search?
+        Disallow: /orkut/albums
+        Allow: /jsapi
+        Disallow: /views?
+        Disallow: /c/
+        Disallow: /cbk
+        Allow: /cbk?output=tile&cb_client=maps_sv
+        Disallow: /kh
+        Disallow: /vt
+        Disallow: /maps/vt
+        Disallow: /recharge/dashboard/car
+        Disallow: /recharge/dashboard/static/
+        Disallow: /translate_a/
+        Disallow: /translate_c
+        Disallow: /translate_f
+        Disallow: /translate_static/
+        Disallow: /translate_suggestion
+        Disallow: /profiles/me
+        Allow: /profiles
+        Disallow: /s2/profiles/me
+        Allow: /s2/profiles
+        Allow: /s2/oz
+        Allow: /s2/photos
+        Allow: /s2/search/social
+        Allow: /s2/static
+        Disallow: /s2
+        Disallow: /transconsole/portal/
+        Disallow: /gcc/
+        Disallow: /aclk
+        Disallow: /cse?
+        Disallow: /cse/home
+        Disallow: /cse/panel
+        Disallow: /cse/manage
+        Disallow: /tbproxy/
+        Disallow: /imesync/
+        Disallow: /shenghuo/search?
+        Disallow: /support/forum/search?
+        Disallow: /reviews/polls/
+        Disallow: /hosted/images/
+        Disallow: /ppob/?
+        Disallow: /ppob?
+        Disallow: /adwordsresellers
+        Disallow: /accounts/ClientLogin
+        Disallow: /accounts/ClientAuth
+        Disallow: /accounts/o8
+        Allow: /accounts/o8/id
+        Disallow: /topicsearch?q=
+        Disallow: /xfx7/
+        Disallow: /squared/api
+        Disallow: /squared/search
+        Disallow: /squared/table
+        Disallow: /toolkit/
+        Allow: /toolkit/*.html
+        Disallow: /globalmarketfinder/
+        Allow: /globalmarketfinder/*.html
+        Disallow: /qnasearch?
+        Disallow: /app/updates
+        Disallow: /sidewiki/entry/
+        Disallow: /quality_form?
+        Disallow: /labs/popgadget/search
+        Disallow: /buzz/post
+        Disallow: /compressiontest/
+        Disallow: /analytics/reporting/
+        Disallow: /analytics/admin/
+        Disallow: /analytics/web/
+        Disallow: /analytics/feeds/
+        Disallow: /analytics/settings/
+        Allow: /alerts/manage
+        Allow: /alerts/remove
+        Disallow: /alerts/
+        Allow: /alerts/$
+        Disallow: /ads/search?
+        Disallow: /ads/plan/action_plan?
+        Disallow: /ads/plan/api/
+        Disallow: /ads/hotels/partners
+        Disallow: /phone/compare/?
+        Disallow: /travel/clk
+        Disallow: /hotelfinder/rpc
+        Disallow: /hotels/rpc
+        Disallow: /flights/rpc
+        Disallow: /commercesearch/services/
+        Disallow: /evaluation/
+        Disallow: /chrome/browser/mobile/tour
+        Disallow: /compare/*/apply*
+        Disallow: /forms/perks/
+        Disallow: /baraza/*/search
+        Disallow: /baraza/*/report
+        Disallow: /shopping/suppliers/search
+        Disallow: /ct/
+        Disallow: /edu/cs4hs/
+        Disallow: /trustedstores/s/
+        Disallow: /trustedstores/tm2
+        Disallow: /trustedstores/verify
+        Disallow: /adwords/proposal
+        Disallow: /shopping/product/
+        Disallow: /shopping/seller
+        Disallow: /shopping/reviewer
+        Disallow: /about/careers/apply/
+        Disallow: /about/careers/applications/
+        Disallow: /landing/signout.html
+        Disallow: /webmasters/sitemaps/ping?
+        Disallow: /ping?
+        Allow: /gb/images
+        Allow: /gb/js
+        Disallow: /gallery/
+        Allow: /chromecast/setup$
+        Allow: /chromecast/setup/$
+        Disallow: /landing/now/ontap/
+        Sitemap: http://www.gstatic.com/culturalinstitute/sitemaps/www_google_com_culturalinstitute/sitemap-index.xml
+        Sitemap: http://www.gstatic.com/dictionary/static/sitemaps/sitemap_index.xml
+        Sitemap: http://www.gstatic.com/earth/gallery/sitemaps/sitemap.xml
+        Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml
+        Sitemap: http://www.gstatic.com/trends/websites/sitemaps/sitemapindex.xml
+        Sitemap: https://www.google.com/sitemap.xml
+
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:15 GMT
+- request:
+    method: get
+    uri: https://www.yahoo.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBot
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 04 Jan 2016 13:39:16 GMT
+      P3p:
+      - policyref="http://info.yahoo.com/w3c/p3p.xml", CP="CAO DSP COR CUR ADM DEV
+        TAI PSA PSD IVAi IVDi CONi TELo OTPi OUR DELi SAMi OTRi UNRi PUBi IND PHY
+        ONL UNI PUR FIN COM NAV INT DEM CNT STA POL HEA PRE LOC GOV"
+      Cache-Control:
+      - private
+      Last-Modified:
+      - Thu, 19 Nov 2015 00:31:02 GMT
+      Accept-Ranges:
+      - bytes
+      Vary:
+      - Accept-Encoding
+      Content-Type:
+      - text/plain; charset=utf-8
+      Age:
+      - '2'
+      Via:
+      - http/1.1 usproxy2.fp.gq1.yahoo.com (ApacheTrafficServer), http/1.1 ir2.fp.sg3.yahoo.com
+        (ApacheTrafficServer)
+      Server:
+      - ATS
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Y-Trace:
+      - BAEAQAAAAAADb4_wm_pNEAAAAAAAAAAAGXeCK6w0SUwAAAAAAAAAAAAFKII_Uu55AAUogj9WSEhTfIBlAAAAAA--
+    body:
+      encoding: ASCII-8BIT
+      string: |
+        User-agent: *
+        Disallow: /p/
+        Disallow: /r/
+        Disallow: /bin/
+        Disallow: /includes/
+        Disallow: /blank.html
+        Disallow: /_td_api
+        Disallow: /_tdpp_api
+        Disallow: /_remote
+        Disallow: /_multiremote
+        Disallow: /_tdhl_api
+
+        Sitemap: https://www.yahoo.com/food/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/tech/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/travel/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/movies/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/beauty/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/health/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/style/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/makers/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/parenting/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/music/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/tv/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/politics/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/autos/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/katiecouric/sitemaps/sitemap_index_us_en-US.xml.gz
+        Sitemap: https://www.yahoo.com/digest/sitemap.xml
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:17 GMT
+- request:
+    method: get
+    uri: http://www.apple.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBot
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Last-Modified:
+      - Thu, 22 Oct 2015 00:20:47 GMT
+      Server:
+      - Apache
+      Nncoection:
+      - close
+      Accept-Ranges:
+      - bytes
+      Content-Length:
+      - '6902'
+      Content-Type:
+      - text/plain; charset=UTF-8
+      Cache-Control:
+      - max-age=561
+      Expires:
+      - Mon, 04 Jan 2016 13:48:38 GMT
+      Date:
+      - Mon, 04 Jan 2016 13:39:17 GMT
+      Connection:
+      - keep-alive
+    body:
+      encoding: UTF-8
+      string: "# robots.txt for http://www.apple.com/\n\nUser-agent: *\nDisallow:
+        /*/includes/*\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/answer/vote*\nDisallow: /*/shop/bag*\nDisallow:
+        /*/shop/browse/campaigns/mobile_overlay*\nDisallow: /*/shop/button_availability*\nDisallow:
+        /*/shop/favorites*\nDisallow: /*/shop/mobile/olss_error*\nDisallow: /*/shop/mobilex/*\nDisallow:
+        /*/shop/order/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow:
+        /*/shop/question/unsubscribe/\nDisallow: /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow:
+        /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/store/feeds/*\nDisallow: /*/shop/variationSelection\nDisallow: /*_adc_*/shop/\nDisallow:
+        /*_aoc_*/shop/\nDisallow: /*_edu_*/shop/\nDisallow: /*_enterprise*/shop/\nDisallow:
+        /*_internal-epp-discounted*/shop/\nDisallow: /*_k12nonbts*/shop/\nDisallow:
+        /*_kiosk*/shop/\nDisallow: /*_nonbts*/shop/\nDisallow: /*_qpromo*/shop/\nDisallow:
+        /*_refurb-discounted*/shop/\nDisallow: /cn/se/*\nDisallow: /retail/availability*\nDisallow:
+        /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow:
+        /search/*\nDisallow: /shop/*signed_in_account*\nDisallow: /shop/*sign_in*\nDisallow:
+        /shop/*sign_out*\nDisallow: /shop/*WebObjects/*\nDisallow: /shop/1-800-MY-APPLE/*\nDisallow:
+        /shop/answer/vote*\nDisallow: /shop/bag*\nDisallow: /shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /shop/button_availability*\nDisallow: /shop/favorites*\nDisallow: /shop/mobile/olss_error*\nDisallow:
+        /shop/mobilex/\nDisallow: /shop/order/*\nDisallow: /shop/question/answer/report*\nDisallow:
+        /shop/question/subscribe*\nDisallow: /shop/question/unsubscribe/\nDisallow:
+        /shop/review/vote*\nDisallow: /shop/reviews/report*\nDisallow: /shop/rs-mvt/rel/*\nDisallow:
+        /shop/search/*\nDisallow: /shop/sentry*\nDisallow: /shop/socialsharing/*\nDisallow:
+        /shop/store/feeds/*\nDisallow: /shop/variationSelection*\nDisallow: /tmall*\nAllow:
+        /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent:
+        Baiduspider\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow:
+        /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /*/shop/browse/overlay*\nDisallow: /*/shop/button_availability*\nDisallow:
+        /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow: /*/shop/checkout*\nDisallow:
+        /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow: /*/shop/identify_user*\nDisallow:
+        /*/shop/mobile/checkout/start*\nDisallow: /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow:
+        /*/shop/question/subscribe*\nDisallow: /*/shop/question/unsubscribe/*\nDisallow:
+        /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow:
+        /*/shop/sentry*\nDisallow: /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow:
+        /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow:
+        /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/storeConfig*\nDisallow: /*/shop/vieworder*\nDisallow: /*/variationSelection*\nDisallow:
+        /ae/*\nDisallow: /am/*\nDisallow: /asia/*\nDisallow: /at/*\nDisallow: /au/*\nDisallow:
+        /befr/*\nDisallow: /benl/*\nDisallow: /bg/*\nDisallow: /bh/*\nDisallow: /br/*\nDisallow:
+        /bw/*\nDisallow: /ca/*\nDisallow: /ca/fr/*\nDisallow: /cf/*\nDisallow: /chde/*\nDisallow:
+        /chfr/*\nDisallow: /ci/*\nDisallow: /cl/*\nDisallow: /cm/*\nDisallow: /cn/se/*\nDisallow:
+        /co/*\nDisallow: /cz/*\nDisallow: /de/*\nDisallow: /dk/*\nDisallow: /ee/*\nDisallow:
+        /eg/*\nDisallow: /es/*\nDisallow: /fi/*\nDisallow: /fr/*\nDisallow: /gn/*\nDisallow:
+        /gq/*\nDisallow: /gr/*\nDisallow: /gw/*\nDisallow: /hk/*\nDisallow: /hk/en/*\nDisallow:
+        /hr/*\nDisallow: /hu/*\nDisallow: /id/*\nDisallow: /ie/*\nDisallow: /il/*\nDisallow:
+        /in/*\nDisallow: /it/*\nDisallow: /jo/*\nDisallow: /jp/*\nDisallow: /ke/*\nDisallow:
+        /kr/*\nDisallow: /kw/*\nDisallow: /la/*\nDisallow: /lae/*\nDisallow: /li/*\nDisallow:
+        /lt/*\nDisallow: /lu/*\nDisallow: /lv/*\nDisallow: /ma/*\nDisallow: /md/*\nDisallow:
+        /me/*\nDisallow: /mg/*\nDisallow: /mk/*\nDisallow: /ml/*\nDisallow: /mt/*\nDisallow:
+        /mu/*\nDisallow: /mx/*\nDisallow: /my/*\nDisallow: /mz/*\nDisallow: /ne/*\nDisallow:
+        /ng/*\nDisallow: /nl/*\nDisallow: /no/*\nDisallow: /nz/*\nDisallow: /om/*\nDisallow:
+        /ph/*\nDisallow: /pl/*\nDisallow: /pt/*\nDisallow: /qa/*\nDisallow: /retail/availability*\nDisallow:
+        /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow:
+        /ro/*\nDisallow: /ru/*\nDisallow: /sa/*\nDisallow: /se/*\nDisallow: /search/*\nDisallow:
+        /sg/*\nDisallow: /shop/*\nDisallow: /si/*\nDisallow: /sk/*\nDisallow: /sn/*\nDisallow:
+        /th/*\nDisallow: /tmall/*\nDisallow: /tn/*\nDisallow: /tr/*\nDisallow: /tw/*\nDisallow:
+        /ug/*\nDisallow: /uk/*\nDisallow: /vn/*\nDisallow: /za/*\n\nUser-agent: HaoSouSpider\nDisallow:
+        /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow:
+        /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /*/shop/button_availability*\nDisallow: /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow:
+        /*/shop/checkout*\nDisallow: /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow:
+        /*/shop/identify_user*\nDisallow: /*/shop/mobile/checkout/start*\nDisallow:
+        /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow:
+        /*/shop/question/unsubscribe/*\nDisallow: /*/shop/review/vote*\nDisallow:
+        /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow:
+        /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow:
+        /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow:
+        /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/storeConfig*\nDisallow: /*/shop/variationSelection*\nDisallow: /*/shop/vieworder*\nDisallow:
+        /cn/se/*\nDisallow: /retail/availability*\nDisallow: /retail/availabilitySearch*\nDisallow:
+        /retail/pickupEligibility*\nDisallow: /search/*\nDisallow: /shop/bag*\nDisallow:
+        /shop/search/*\nDisallow: /tmall/*\n\nSitemap: http://www.apple.com/shop/sitemap.xml\nSitemap:
+        http://www.apple.com/sitemap.xml\n"
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:17 GMT
+- request:
+    method: get
+    uri: http://www.apple.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBaiduspider
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Last-Modified:
+      - Thu, 22 Oct 2015 00:20:47 GMT
+      Server:
+      - Apache
+      Nncoection:
+      - close
+      Accept-Ranges:
+      - bytes
+      Content-Length:
+      - '6902'
+      Content-Type:
+      - text/plain; charset=UTF-8
+      Cache-Control:
+      - max-age=417
+      Expires:
+      - Mon, 04 Jan 2016 13:46:14 GMT
+      Date:
+      - Mon, 04 Jan 2016 13:39:17 GMT
+      Connection:
+      - keep-alive
+    body:
+      encoding: UTF-8
+      string: "# robots.txt for http://www.apple.com/\n\nUser-agent: *\nDisallow:
+        /*/includes/*\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/answer/vote*\nDisallow: /*/shop/bag*\nDisallow:
+        /*/shop/browse/campaigns/mobile_overlay*\nDisallow: /*/shop/button_availability*\nDisallow:
+        /*/shop/favorites*\nDisallow: /*/shop/mobile/olss_error*\nDisallow: /*/shop/mobilex/*\nDisallow:
+        /*/shop/order/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow:
+        /*/shop/question/unsubscribe/\nDisallow: /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow:
+        /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/store/feeds/*\nDisallow: /*/shop/variationSelection\nDisallow: /*_adc_*/shop/\nDisallow:
+        /*_aoc_*/shop/\nDisallow: /*_edu_*/shop/\nDisallow: /*_enterprise*/shop/\nDisallow:
+        /*_internal-epp-discounted*/shop/\nDisallow: /*_k12nonbts*/shop/\nDisallow:
+        /*_kiosk*/shop/\nDisallow: /*_nonbts*/shop/\nDisallow: /*_qpromo*/shop/\nDisallow:
+        /*_refurb-discounted*/shop/\nDisallow: /cn/se/*\nDisallow: /retail/availability*\nDisallow:
+        /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow:
+        /search/*\nDisallow: /shop/*signed_in_account*\nDisallow: /shop/*sign_in*\nDisallow:
+        /shop/*sign_out*\nDisallow: /shop/*WebObjects/*\nDisallow: /shop/1-800-MY-APPLE/*\nDisallow:
+        /shop/answer/vote*\nDisallow: /shop/bag*\nDisallow: /shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /shop/button_availability*\nDisallow: /shop/favorites*\nDisallow: /shop/mobile/olss_error*\nDisallow:
+        /shop/mobilex/\nDisallow: /shop/order/*\nDisallow: /shop/question/answer/report*\nDisallow:
+        /shop/question/subscribe*\nDisallow: /shop/question/unsubscribe/\nDisallow:
+        /shop/review/vote*\nDisallow: /shop/reviews/report*\nDisallow: /shop/rs-mvt/rel/*\nDisallow:
+        /shop/search/*\nDisallow: /shop/sentry*\nDisallow: /shop/socialsharing/*\nDisallow:
+        /shop/store/feeds/*\nDisallow: /shop/variationSelection*\nDisallow: /tmall*\nAllow:
+        /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent:
+        Baiduspider\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow:
+        /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /*/shop/browse/overlay*\nDisallow: /*/shop/button_availability*\nDisallow:
+        /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow: /*/shop/checkout*\nDisallow:
+        /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow: /*/shop/identify_user*\nDisallow:
+        /*/shop/mobile/checkout/start*\nDisallow: /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow:
+        /*/shop/question/subscribe*\nDisallow: /*/shop/question/unsubscribe/*\nDisallow:
+        /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow:
+        /*/shop/sentry*\nDisallow: /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow:
+        /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow:
+        /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/storeConfig*\nDisallow: /*/shop/vieworder*\nDisallow: /*/variationSelection*\nDisallow:
+        /ae/*\nDisallow: /am/*\nDisallow: /asia/*\nDisallow: /at/*\nDisallow: /au/*\nDisallow:
+        /befr/*\nDisallow: /benl/*\nDisallow: /bg/*\nDisallow: /bh/*\nDisallow: /br/*\nDisallow:
+        /bw/*\nDisallow: /ca/*\nDisallow: /ca/fr/*\nDisallow: /cf/*\nDisallow: /chde/*\nDisallow:
+        /chfr/*\nDisallow: /ci/*\nDisallow: /cl/*\nDisallow: /cm/*\nDisallow: /cn/se/*\nDisallow:
+        /co/*\nDisallow: /cz/*\nDisallow: /de/*\nDisallow: /dk/*\nDisallow: /ee/*\nDisallow:
+        /eg/*\nDisallow: /es/*\nDisallow: /fi/*\nDisallow: /fr/*\nDisallow: /gn/*\nDisallow:
+        /gq/*\nDisallow: /gr/*\nDisallow: /gw/*\nDisallow: /hk/*\nDisallow: /hk/en/*\nDisallow:
+        /hr/*\nDisallow: /hu/*\nDisallow: /id/*\nDisallow: /ie/*\nDisallow: /il/*\nDisallow:
+        /in/*\nDisallow: /it/*\nDisallow: /jo/*\nDisallow: /jp/*\nDisallow: /ke/*\nDisallow:
+        /kr/*\nDisallow: /kw/*\nDisallow: /la/*\nDisallow: /lae/*\nDisallow: /li/*\nDisallow:
+        /lt/*\nDisallow: /lu/*\nDisallow: /lv/*\nDisallow: /ma/*\nDisallow: /md/*\nDisallow:
+        /me/*\nDisallow: /mg/*\nDisallow: /mk/*\nDisallow: /ml/*\nDisallow: /mt/*\nDisallow:
+        /mu/*\nDisallow: /mx/*\nDisallow: /my/*\nDisallow: /mz/*\nDisallow: /ne/*\nDisallow:
+        /ng/*\nDisallow: /nl/*\nDisallow: /no/*\nDisallow: /nz/*\nDisallow: /om/*\nDisallow:
+        /ph/*\nDisallow: /pl/*\nDisallow: /pt/*\nDisallow: /qa/*\nDisallow: /retail/availability*\nDisallow:
+        /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow:
+        /ro/*\nDisallow: /ru/*\nDisallow: /sa/*\nDisallow: /se/*\nDisallow: /search/*\nDisallow:
+        /sg/*\nDisallow: /shop/*\nDisallow: /si/*\nDisallow: /sk/*\nDisallow: /sn/*\nDisallow:
+        /th/*\nDisallow: /tmall/*\nDisallow: /tn/*\nDisallow: /tr/*\nDisallow: /tw/*\nDisallow:
+        /ug/*\nDisallow: /uk/*\nDisallow: /vn/*\nDisallow: /za/*\n\nUser-agent: HaoSouSpider\nDisallow:
+        /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow:
+        /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow:
+        /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow:
+        /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow:
+        /*/shop/button_availability*\nDisallow: /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow:
+        /*/shop/checkout*\nDisallow: /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow:
+        /*/shop/identify_user*\nDisallow: /*/shop/mobile/checkout/start*\nDisallow:
+        /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow:
+        /*/shop/question/unsubscribe/*\nDisallow: /*/shop/review/vote*\nDisallow:
+        /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow:
+        /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow:
+        /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow:
+        /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow:
+        /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow:
+        /*/shop/storeConfig*\nDisallow: /*/shop/variationSelection*\nDisallow: /*/shop/vieworder*\nDisallow:
+        /cn/se/*\nDisallow: /retail/availability*\nDisallow: /retail/availabilitySearch*\nDisallow:
+        /retail/pickupEligibility*\nDisallow: /search/*\nDisallow: /shop/bag*\nDisallow:
+        /shop/search/*\nDisallow: /tmall/*\n\nSitemap: http://www.apple.com/shop/sitemap.xml\nSitemap:
+        http://www.apple.com/sitemap.xml\n"
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:17 GMT
+- request:
+    method: get
+    uri: https://github.com/robots.txt
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - TestBot
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - GitHub.com
+      Date:
+      - Mon, 04 Jan 2016 13:39:18 GMT
+      Content-Type:
+      - text/plain
+      Last-Modified:
+      - Mon, 04 Jan 2016 13:28:38 GMT
+      Transfer-Encoding:
+      - chunked
+      Etag:
+      - W/"568a7386-6e36"
+      Vary:
+      - Accept-Encoding
+      X-Served-By:
+      - a568c03544f42dddf712bab3bfd562fd
+      X-Github-Request-Id:
+      - 7C27244A:448A:3EB799EF:568A7605
+    body:
+      encoding: ASCII-8BIT
+      string: |
+        # If you would like to crawl GitHub contact us at support@github.com.
+        # We also provide an extensive API: https://developer.github.com/
+
+        User-agent: CCBot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: coccoc
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: dotbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: duckduckbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: EtaoSpider
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Googlebot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: HTTrack
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: ia_archiver
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: IntuitGSACrawler
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Mail.RU_Bot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: msnbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Bingbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: naverbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: red-app-gsa-p-one
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: rogerbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: SandDollar
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: seznambot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Slurp
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Swiftbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Telefonica
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: teoma
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Twitterbot
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+        User-agent: Yandex
+        Allow: /*/*/tree/master
+        Allow: /*/*/blob/master
+        Disallow: /ekansa/Open-Context-Data
+        Disallow: /ekansa/opencontext-*
+        Disallow: /*/*/pulse
+        Disallow: /*/*/tree/*
+        Disallow: /*/*/blob/*
+        Disallow: /*/*/wiki/*/*
+        Disallow: /gist/*/*/*
+        Disallow: /oembed
+        Disallow: /*/forks
+        Disallow: /*/stars
+        Disallow: /*/download
+        Disallow: /*/revisions
+        Disallow: /*/*/issues/new
+        Disallow: /*/*/issues/search
+        Disallow: /*/*/commits/*/*
+        Disallow: /*/*/commits/*?author
+        Disallow: /*/*/commits/*?path
+        Disallow: /*/*/branches
+        Disallow: /*/*/tags
+        Disallow: /*/*/contributors
+        Disallow: /*/*/comments
+        Disallow: /*/*/stargazers
+        Disallow: /*/*/search
+        Disallow: /*/tarball/
+        Disallow: /*/zipball/
+        Disallow: /*/*/archive/
+        Disallow: /raw/*
+        Disallow: /*/followers
+        Disallow: /*/following
+        Disallow: /stars/*
+        Disallow: /*/blame/
+        Disallow: /*/watchers
+        Disallow: /*/network
+        Disallow: /*/graphs
+        Disallow: /*/raw/
+        Disallow: /*/compare/
+        Disallow: /*/cache/
+        Disallow: /*/*/blame/
+        Disallow: /*/*/watchers
+        Disallow: /*/*/network
+        Disallow: /*/*/graphs
+        Disallow: /*/*/raw/
+        Disallow: /*/*/compare/
+        Disallow: /*/*/cache/
+        Disallow: /*.git
+        Disallow: /*.git/
+        Disallow: /search/advanced
+        Disallow: /search
+        Disallow: /*q=
+        Disallow: /*.atom
+        Disallow: /login
+
+
+        User-agent: *
+        Allow: /humans.txt
+        Disallow: /
+    http_version: 
+  recorded_at: Mon, 04 Jan 2016 13:39:18 GMT
+recorded_with: VCR 3.0.0