device_detector 0.8.2 → 0.9.0

data/spec/fixtures/parser/vendorfragments.yml

@@ -25,6 +25,12 @@
 -
   useragent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; CMNTDF; InfoPath.2; .NET4.0C; .NET4.0E)
   vendor: CQ
+-
+  user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.2; WOW64; Trident/6.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729; CMDTDFJS; Tablet PC 2.0)
+  vendor: CQ
+-
+  user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; CMDTDF; InfoPath.3)
+  vendor: CQ
 -
   useragent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Win64; x64; Trident/4.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MDDC; .NET4.0C; .NET4.0E)
   vendor: DL

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: device_detector
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.9.0
 platform: ruby
 authors:
 - Mati Sójka
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-20 00:00:00.000000000 Z
+date: 2016-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -126,20 +126,25 @@ files:
 - spec/fixtures/detector/phablet.yml
 - spec/fixtures/detector/portable_media_player.yml
 - spec/fixtures/detector/smart_display.yml
+- spec/fixtures/detector/smartphone-1.yml
+- spec/fixtures/detector/smartphone-2.yml
+- spec/fixtures/detector/smartphone-3.yml
+- spec/fixtures/detector/smartphone-4.yml
 - spec/fixtures/detector/smartphone.yml
+- spec/fixtures/detector/tablet-1.yml
+- spec/fixtures/detector/tablet-2.yml
 - spec/fixtures/detector/tablet.yml
 - spec/fixtures/detector/tv.yml
 - spec/fixtures/detector/unknown.yml
 - spec/fixtures/device/camera.yml
 - spec/fixtures/device/car_browser.yml
 - spec/fixtures/device/console.yml
-- spec/fixtures/parser/bots.yml
 - spec/fixtures/parser/oss.yml
 - spec/fixtures/parser/vendorfragments.yml
 - spec/spec_helper.rb
 homepage: http://podigee.github.io/device_detector
 licenses:
-- LGPLv3
+- LGPL-3.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -157,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Precise and fast user agent parser and device detector
@@ -191,14 +196,19 @@ test_files:
 - spec/fixtures/detector/phablet.yml
 - spec/fixtures/detector/portable_media_player.yml
 - spec/fixtures/detector/smart_display.yml
+- spec/fixtures/detector/smartphone-1.yml
+- spec/fixtures/detector/smartphone-2.yml
+- spec/fixtures/detector/smartphone-3.yml
+- spec/fixtures/detector/smartphone-4.yml
 - spec/fixtures/detector/smartphone.yml
+- spec/fixtures/detector/tablet-1.yml
+- spec/fixtures/detector/tablet-2.yml
 - spec/fixtures/detector/tablet.yml
 - spec/fixtures/detector/tv.yml
 - spec/fixtures/detector/unknown.yml
 - spec/fixtures/device/camera.yml
 - spec/fixtures/device/car_browser.yml
 - spec/fixtures/device/console.yml
-- spec/fixtures/parser/bots.yml
 - spec/fixtures/parser/oss.yml
 - spec/fixtures/parser/vendorfragments.yml
 - spec/spec_helper.rb

data/spec/fixtures/parser/bots.yml

@@ -1,2055 +0,0 @@
--
-  user_agent: Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider
-  bot:
-    name: '360Spider'
-    category: 'Search bot'
-    url: 'http://www.so.com/help/help_3_2.html'
-    producer:
-      name: 'Online Media Group, Inc.'
-      url: ''
--
-  user_agent: Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider
-  bot:
-    name: '360Spider'
-    category: 'Search bot'
-    url: 'http://www.so.com/help/help_3_2.html'
-    producer:
-      name: 'Online Media Group, Inc.'
-      url: ''
--
-  user_agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MDDR; .NET4.0C; .NET4.0E; .NET CLR 1.1.4322; Tablet PC 2.0); 360Spider
-  bot:
-    name: '360Spider'
-    category: 'Search bot'
-    url: 'http://www.so.com/help/help_3_2.html'
-    producer:
-      name: 'Online Media Group, Inc.'
-      url: ''
--
-  user_agent: 360spider-image
-  bot:
-    name: '360Spider'
-    category: 'Search bot'
-    url: 'http://www.so.com/help/help_3_2.html'
-    producer:
-      name: 'Online Media Group, Inc.'
-      url: ''
--
-  user_agent: Aboundex/0.3 (http://www.aboundex.com/crawler/)
-  bot:
-    name: 'Aboundexbot'
-    category: 'Search bot'
-    url: 'http://www.aboundex.com/crawler/'
-    producer:
-      name: 'Aboundex.com'
-      url: 'http://www.aboundex.com'
--
-  user_agent: Mozilla/5.0 (compatible; AcoonBot/4.11.1; +http://www.acoon.de/robot.asp)
-  bot:
-    name: 'Acoon'
-    category: 'Search bot'
-    url: 'http://www.acoon.de/robot.asp'
-    producer:
-      name: 'Acoon GmbH'
-      url: 'http://www.acoon.de'
--
-  user_agent: AddThis.com robot tech.support@clearspring.com
-  bot:
-    name: 'AddThis.com'
-    category: 'Social Media Agent'
-    url: ''
-    producer:
-      name: 'Clearspring Technologies, Inc.'
-      url: 'http://www.clearspring.com'
--
-  user_agent: Mozilla/5.0 (compatible; AhrefsBot/3.1; +http://ahrefs.com/robot/
-  bot:
-    name: 'aHrefs Bot'
-    category: 'Crawler'
-    url: 'http://ahrefs.com/robot'
-    producer:
-      name: 'Ahrefs Pte Ltd'
-      url: 'http://ahrefs.com/robot'
--
-  user_agent: Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)
-  bot:
-    name: 'aHrefs Bot'
-    category: 'Crawler'
-    url: 'http://ahrefs.com/robot'
-    producer:
-      name: 'Ahrefs Pte Ltd'
-      url: 'http://ahrefs.com/robot'
--
-  user_agent: 'Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/) AppEngine-Google; (+http://code.google.com/appengine; appid: s~proxyfile1-hrd)'
-  bot:
-    name: 'aHrefs Bot'
-    category: 'Crawler'
-    url: 'http://ahrefs.com/robot'
-    producer:
-      name: 'Ahrefs Pte Ltd'
-      url: 'http://ahrefs.com/robot'
--
-  user_agent: AhrefsBot.Feeds v0.1; http://ahrefs.com/
-  bot:
-    name: 'aHrefs Bot'
-    category: 'Crawler'
-    url: 'http://ahrefs.com/robot'
-    producer:
-      name: 'Ahrefs Pte Ltd'
-      url: 'http://ahrefs.com/robot'
--
-  user_agent: ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)
-  bot:
-    name: 'Alexa Crawler'
-    category: 'Search bot'
-    url: 'https://alexa.zendesk.com/hc/en-us/sections/200100794-Crawlers'
-    producer:
-      name: 'Alexa Internet'
-      url: 'http://www.alexa.com'
--
-  user_agent: AmorankSpider/0.1; +http://amorank.com/webcrawler.html
-  bot:
-    name: 'Amorank Spider'
-    category: 'Crawler'
-    url: 'http://amorank.com/webcrawler.html'
-    producer:
-      name: 'Amorank'
-      url: 'http://www.amorank.com'
--
-  user_agent: Curious George - www.analyticsseo.com/crawler
-  bot:
-    name: 'Analytics SEO Crawler'
-    category: 'Crawler'
-    url: 'http://www.analyticsseo.com/crawler'
-    producer:
-      name: 'Analytics SEO'
-      url: 'http://www.analyticsseo.com'
--
-  user_agent: Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)
-  bot:
-    name: 'archive.org bot'
-    category: 'Crawler'
-    url: 'http://www.archive.org/details/archive.org_bot'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; special_archiver/3.2.0 +http://www.loc.gov/webarchiving/notice_to_webmasters.html)
-  bot:
-    name: 'archive.org bot'
-    category: 'Crawler'
-    url: 'http://www.archive.org/details/archive.org_bot'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)
-  bot:
-    name: 'archive.org bot'
-    category: 'Crawler'
-    url: 'http://www.archive.org/details/archive.org_bot'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +http://pandora.nla.gov.au/crawl.html)
-  bot:
-    name: 'archive.org bot'
-    category: 'Crawler'
-    url: 'http://www.archive.org/details/archive.org_bot'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; archive.org_bot; Archive-It; +http://archive-it.org/files/site-owners.html)
-  bot:
-    name: 'archive.org bot'
-    category: 'Crawler'
-    url: 'http://www.archive.org/details/archive.org_bot'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)
-  bot:
-    name: 'archive.org bot'
-    category: 'Crawler'
-    url: 'http://www.archive.org/details/archive.org_bot'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20131029-0036 +http://archive.org/details/archive.org_bot)
-  bot:
-    name: 'archive.org bot'
-    category: 'Crawler'
-    url: 'http://www.archive.org/details/archive.org_bot'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
-  bot:
-    name: 'Ask Jeeves'
-    category: 'Search bot'
-    url: ''
-    producer:
-      name: 'Ask Jeeves Inc.'
-      url: 'http://www.ask.com'
--
-  user_agent: Backlink-Ceck.de (+http://www.backlink-check.de/bot.html)
-  bot:
-    name: 'Backlink-Ceck.de'
-    category: 'Crawler'
-    url: 'http://www.backlink-check.de/bot.html'
-    producer:
-      name: 'Mediagreen Medienservice'
-      url: 'http://www.backlink-check.de'
--
-  user_agent: BacklinkCrawler (http://www.backlinktest.com/crawler.html)
-  bot:
-    name: 'BacklinkCrawler'
-    category: 'Crawler'
-    url: 'http://www.backlinktest.com/crawler.html'
-    producer:
-      name: '2.0Promotion GbR'
-      url: 'http://www.backlinktest.com'
--
-  user_agent: Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search
-  bot:
-    name: 'Baidu Spider'
-    category: 'Search bot'
-    url: 'http://www.baidu.com/search/spider.htm'
-    producer:
-      name: 'Baidu'
-      url: 'http://www.baidu.com'
--
-  user_agent: Mozilla/5.0 (Linux;u;Android 2.3.7;zh-cn;) AppleWebKit/533.1 (KHTML,like Gecko) Version/4.0 Mobile Safari/533.1 (compatible; +http://www.baidu.com/search/spider.html)
-  bot:
-    name: 'Baidu Spider'
-    category: 'Search bot'
-    url: 'http://www.baidu.com/search/spider.htm'
-    producer:
-      name: 'Baidu'
-      url: 'http://www.baidu.com'
--
-  user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8;baidu Transcoder) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729)
-  bot:
-    name: 'Baidu Spider'
-    category: 'Search bot'
-    url: 'http://www.baidu.com/search/spider.htm'
-    producer:
-      name: 'Baidu'
-      url: 'http://www.baidu.com'
--
-  user_agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; baidu Transcoder;)
-  bot:
-    name: 'Baidu Spider'
-    category: 'Search bot'
-    url: 'http://www.baidu.com/search/spider.htm'
-    producer:
-      name: 'Baidu'
-      url: 'http://www.baidu.com'
--
-  user_agent: Baiduspider-image+(+http://www.baidu.com/search/spider.htm)
-  bot:
-    name: 'Baidu Spider'
-    category: 'Search bot'
-    url: 'http://www.baidu.com/search/spider.htm'
-    producer:
-      name: 'Baidu'
-      url: 'http://www.baidu.com'
--
-  user_agent: Baiduspider+(+http://www.baidu.com/search/spider.htm)
-  bot:
-    name: 'Baidu Spider'
-    category: 'Search bot'
-    url: 'http://www.baidu.com/search/spider.htm'
-    producer:
-      name: 'Baidu'
-      url: 'http://www.baidu.com'
--
-  user_agent: Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-  bot:
-    name: 'Baidu Spider'
-    category: 'Search bot'
-    url: 'http://www.baidu.com/search/spider.htm'
-    producer:
-      name: 'Baidu'
-      url: 'http://www.baidu.com'
--
-  user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b
-  bot:
-    name: 'BingBot'
-    category: 'Search bot'
-    url: 'http://search.msn.com/msnbot.htmn'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
-  bot:
-    name: 'BingBot'
-    category: 'Search bot'
-    url: 'http://search.msn.com/msnbot.htmn'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)
-  bot:
-    name: 'BingBot'
-    category: 'Search bot'
-    url: 'http://search.msn.com/msnbot.htmn'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: MSNBot/Nutch-1.5.1
-  bot:
-    name: 'BingBot'
-    category: 'Search bot'
-    url: 'http://search.msn.com/msnbot.htmn'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: msnbot/2.0b v1394720947.6703
-  bot:
-    name: 'BingBot'
-    category: 'Search bot'
-    url: 'http://search.msn.com/msnbot.htmn'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: msnbot/1.1 (+http://search.msn.com/msnbot.htm)
-  bot:
-    name: 'BingBot'
-    category: 'Search bot'
-    url: 'http://search.msn.com/msnbot.htmn'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: msnbot/2.0b (+http://search.msn.com/msnbot.htm)
-  bot:
-    name: 'BingBot'
-    category: 'Search bot'
-    url: 'http://search.msn.com/msnbot.htmn'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)
-  bot:
-    name: 'BingBot'
-    category: 'Search bot'
-    url: 'http://search.msn.com/msnbot.htmn'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)
-  bot:
-    name: 'Blekkobot'
-    category: 'Search bot'
-    url: 'http://blekko.com/about/blekkobot'
-    producer:
-      name: 'Blekko'
-      url: 'http://blekko.com'
--
-  user_agent: Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
-  bot:
-    name: 'BLEXBot Crawler'
-    category: 'Crawler'
-    url: 'http://webmeup-crawler.com'
-    producer:
-      name: 'WebMeUp'
-      url: 'http://webmeup.com'
--
-  user_agent: Bloglovin/1.0 (http://www.bloglovin.com; 1 subscribers)
-  bot:
-    name: 'Bloglovin'
-    url: 'http://www.bloglovin.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; BountiiBot/1.1; +http://bountii.com/contact.php)
-  bot:
-    name: 'Bountii Bot'
-    category: 'Search bot'
-    url: 'http://bountii.com/contact.php'
-    producer:
-      name: 'Bountii Inc.'
-      url: 'http://bountii.com'
--
-  user_agent: Mozilla/5.0 (Macintosh; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
-  bot:
-    name: 'Butterfly Robot'
-    category: 'Search bot'
-    url: 'http://labs.topsy.com/butterfly'
-    producer:
-      name: 'Topsy Labs'
-      url: 'http://labs.topsy.com'
--
-  user_agent: Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
-  bot:
-    name: 'Butterfly Robot'
-    category: 'Search bot'
-    url: 'http://labs.topsy.com/butterfly'
-    producer:
-      name: 'Topsy Labs'
-      url: 'http://labs.topsy.com'
--
-  user_agent: Browsershots
-  bot:
-    name: 'Browsershots'
-    category: 'Service Agent'
-    url: 'http://browsershots.org/faq'
-    producer:
-      name: 'Browsershots.org'
-      url: 'http://browsershots.org'
--
-  user_agent: Mozilla/5.0 (compatible; CareerBot/1.1; +http://www.career-x.de/bot.html)
-  bot:
-    name: 'CareerBot'
-    category: 'Crawler'
-    url: 'http://www.career-x.de/bot.html'
-    producer:
-      name: 'career-x GmbH'
-      url: 'http://www.career-x.de'
--
-  user_agent: CCBot/2.0 (http://commoncrawl.org/faq/)
-  bot:
-    name: 'ccBot crawler'
-    category: 'Crawler'
-    url: 'http://commoncrawl.org/faq/'
-    producer:
-      name: 'reddit inc.'
-      url: 'http://www.reddit.com'
--
-  user_agent: Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)
-  bot:
-    name: 'Cliqzbot'
-    category: 'Crawler'
-    url: 'http://cliqz.com/company/cliqzbot'
-    producer:
-      name: '10betterpages GmbH'
-      url: 'http://cliqz.com'
--
-  user_agent: CommaFeed/1.0 (http://www.commafeed.com)
-  bot:
-    name: 'CommaFeed'
-    url: 'http://www.commafeed.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; Dazoobot/0.1; +http://dazoo.fr)
-  bot:
-    name: 'Dazoobot'
-    category: 'Search bot'
-    url: ''
-    producer:
-      name: 'DAZOO.FR'
-      url: 'http://dazoo.fr'
--
-  user_agent: Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)
-  bot:
-    name: 'Discobot'
-    category: 'Search bot'
-    url: 'http://discoveryengine.com/discobot.html'
-    producer:
-      name: 'Discovery Engine'
-      url: 'http://discoveryengine.com'
--
-  user_agent: dotbot
-  bot:
-    name: 'DotBot'
-    category: 'Crawler'
-    url: 'http://www.opensiteexplorer.org/dotbot'
-    producer:
-      name: 'SEOmoz, Inc.'
-      url: 'http://moz.com/'
--
-  user_agent: Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)
-  bot:
-    name: 'Easou Spider'
-    category: 'Search bot'
-    url: 'http://www.easou.com/search/spider.html'
-    producer:
-      name: 'easou ICP'
-      url: 'http://www.easou.com'
--
-  user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us; EasouSpider; +http://www.easou.com/search/spider.html)
-  bot:
-    name: 'Easou Spider'
-    category: 'Search bot'
-    url: 'http://www.easou.com/search/spider.html'
-    producer:
-      name: 'easou ICP'
-      url: 'http://www.easou.com'
--
-  user_agent: EMail Exractor
-  bot:
-    name: 'EMail Exractor'
-    category: 'Crawler'
-    url: ''
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)
-  bot:
-    name: 'ExaBot'
-    category: 'Crawler'
-    url: 'http://www.exabot.com/go/robot'
-    producer:
-      name: 'Dassault Systèmes'
-      url: 'http://www.3ds.com'
--
-  user_agent: Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)
-  bot:
-    name: 'ExaBot'
-    category: 'Crawler'
-    url: 'http://www.exabot.com/go/robot'
-    producer:
-      name: 'Dassault Systèmes'
-      url: 'http://www.3ds.com'
--
-  user_agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)
-  bot:
-    name: 'ExaBot'
-    category: 'Crawler'
-    url: 'http://www.exabot.com/go/robot'
-    producer:
-      name: 'Dassault Systèmes'
-      url: 'http://www.3ds.com'
--
-  user_agent: ExactSeek Crawler (nutch 1.4)/Nutch-1.4 (ExactSeek Crawler; http://www.exactseek.com)
-  bot:
-    name: 'ExactSeek Crawler'
-    category: 'Search bot'
-    url: 'http://www.exactseek.com'
-    producer:
-      name: 'Jayde Online, Inc.'
-      url: 'http://www.jaydeonlineinc.com'
--
-  user_agent: Mozilla/5.0 (compatible; Ezooms/1.0; help@moz.com)
-  bot:
-    name: 'Ezooms'
-    category: 'Crawler'
-    url: ''
-    producer:
-      name: 'SEOmoz, Inc.'
-      url: 'http://moz.com/'
--
-  user_agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
-  bot:
-    name: 'Facebook External Hit'
-    category: 'Social Media Agent'
-    url: 'https://www.facebook.com/externalhit_uatext.php'
-    producer:
-      name: 'Facebook'
-      url: 'http://www.facebook.com'
--
-  user_agent: facebookplatform/1.0 (+http://developers.facebook.com)
-  bot:
-    name: 'Facebook External Hit'
-    category: 'Social Media Agent'
-    url: 'https://www.facebook.com/externalhit_uatext.php'
-    producer:
-      name: 'Facebook'
-      url: 'http://www.facebook.com'
--
-  user_agent: Feedbin - 9 subscribers
-  bot:
-    name: 'Feedbin'
-    url: 'http://feedbin.com/'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: FeedBurner/1.0 (http://www.FeedBurner.com)
-  bot:
-    name: 'FeedBurner'
-    url: 'http://www.feedburner.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)
-  bot:
-    name: 'Feedly'
-    url: 'http://www.feedly.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: FeedlyBot/1.0 (http://feedly.com)
-  bot:
-    name: 'Feedly'
-    url: 'http://www.feedly.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: FeedlyApp/1.0 (http://www.feedly.com)
-  bot:
-    name: 'Feedly'
-    url: 'http://www.feedly.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: MetaFeedly/1.0 (http://www.feedly.com)
-  bot:
-    name: 'Feedly'
-    url: 'http://www.feedly.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Feedspot http://www.feedspot.com
-  bot:
-    name: 'Feedspot'
-    url: 'http://www.feedspot.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)
-  bot:
-    name: 'Fever'
-    url: 'http://feedafever.com/'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)
-  bot:
-    name: 'Genieo Web filter'
-    category: ''
-    url: 'http://www.genieo.com/webfilter.html'
-    producer:
-      name: 'Genieo'
-      url: 'http://www.genieo.com'
--
-  user_agent: DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)
-  bot:
-    name: 'Goo'
-    category: 'Search bot'
-    url: 'http://search.goo.ne.jp/option/use/sub4/sub4-1'
-    producer:
-      name: 'NTT Resonant'
-      url: 'http://goo.ne.jp'
--
-  user_agent: Googlebot/2.1 (http://www.googlebot.com/bot.html)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Googlebot/Nutch-1.7
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Googlebot-Image/1.0
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Googlebot (gocrawl v0.4)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Googlebot-News (2.3.3, ruby 1.9.3 (2013-11-22))
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Mozilla/5.0 (compatible) Feedfetcher-Google;(+http://www.google.com/feedfetcher.html)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 2 subscribers; feed-id=17860707833818568603)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 19 subscribers; feed-id=13965549748850348809)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 375 subscribers; feed-id=15381863289700640853)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: GoogleProducer; (+http://goo.gl/7y4SX)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: AdsBot-Google-Mobile (+http://www.google.com/mobile/adsbot.html) Mozilla (iPhone; U; CPU iPhone OS 3 0 like Mac OS X) AppleWebKit (KHTML, like Gecko) Mobile Safari
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: AdsBot-Google (+http://www.google.com/adsbot.html)
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Mediapartners-Google
-  bot:
-    name: 'Googlebot'
-    category: 'Search bot'
-    url: 'http://www.google.com/bot.html'
-    producer:
-      name: 'Google Inc.'
-      url: 'http://www.google.com'
--
-  user_agent: Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.dla-marbach.de)
-  bot:
-    name: 'Heritrix'
-    category: 'Crawler'
-    url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)
-  bot:
-    name: 'Heritrix'
-    category: 'Crawler'
-    url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.baidu.com)
-  bot:
-    name: 'Heritrix'
-    category: 'Crawler'
-    url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: flieder - neofonie heritrix/1.14.3 (+http://spider.neofonie.de)
-  bot:
-    name: 'Heritrix'
-    category: 'Crawler'
-    url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: mozilla/5.0 (compatible; heritrix/1.3.0 http://archive.crawler.org)
-  bot:
-    name: 'Heritrix'
-    category: 'Crawler'
-    url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20130207.001528 +http://webarchiv.cz/kontakty/)
-  bot:
-    name: 'Heritrix'
-    category: 'Crawler'
-    url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; heritrix/1.14.4 +http://webarhive.nlc.gov.cn)
-  bot:
-    name: 'Heritrix'
-    category: 'Crawler'
-    url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.run4dom.com)
-  bot:
-    name: 'Heritrix'
-    category: 'Crawler'
-    url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-    producer:
-      name: 'The Internet Archive'
-      url: 'http://www.archive.org'
--
-  user_agent: HTTPMon/1.0b (http://www.httpmon.com)
-  bot:
-    name: 'HTTPMon'
-    category: 'Site Monitor'
-    url: 'http://www.httpmon.com'
-    producer:
-      name: 'towards GmbH'
-      url: 'http://www.towards.ch/'
--
-  user_agent: iisbot/1.0 (+http://www.iis.net/iisbot.html)
-  bot:
-    name: 'IIS Site Analysis'
-    category: 'crawler'
-    url: 'http://www.iis.net/iisbot.html'
-    producer:
-      name: 'Microsoft Corporation'
-      url: 'http://www.microsoft.com'
--
-  user_agent: kouio.com RSS reader
-  bot:
-    name: 'Kouio'
-    url: 'http://kouio.com/'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: kouio.com RSS reader - 6 subscribers
-  bot:
-    name: 'Kouio'
-    url: 'http://kouio.com/'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; linkdexbot-mobile/2.1; +http://www.linkdex.com/about/bots/)
-  bot:
-    name: 'Linkdex Bot'
-    category: 'Search bot'
-    url: 'http://www.linkdex.com/bots'
-    producer:
-      name: 'Mojeek Ltd.'
-      url: 'http://www.mojeek.com'
--
-  user_agent: Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)
-  bot:
-    name: 'Linkdex Bot'
-    category: 'Search bot'
-    url: 'http://www.linkdex.com/bots'
-    producer:
-      name: 'Mojeek Ltd.'
-      url: 'http://www.mojeek.com'
--
-  user_agent: linkdex.com/v2.0 and linkdex.com/v2.1
-  bot:
-    name: 'Linkdex Bot'
-    category: 'Search bot'
-    url: 'http://www.linkdex.com/bots'
-    producer:
-      name: 'Mojeek Ltd.'
-      url: 'http://www.mojeek.com'
--
-  user_agent: LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)
-  bot:
-    name: 'LinkedIn Bot'
-    category: 'Social Media Agent'
-    url: 'http://www.linkedin.com'
-    producer:
-      name: 'LinkedIn'
-      url: 'http://www.linkedin.com'
--
-  user_agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
-  bot:
-    name: 'Mail.Ru Bot'
-    category: 'Search bot'
-    url: 'http://help.mail.ru/webmaster/indexing/robots/types_robots'
-    producer:
-      name: 'Mail.Ru Group'
-      url: 'http://corp.mail.ru'
--
-  user_agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +http://go.mail.ru/help/robots)
-  bot:
-    name: 'Mail.Ru Bot'
-    category: 'Search bot'
-    url: 'http://help.mail.ru/webmaster/indexing/robots/types_robots'
-    producer:
-      name: 'Mail.Ru Group'
-      url: 'http://corp.mail.ru'
--
-  user_agent: magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)
-  bot:
-    name: 'Magpie-Crawler'
-    category: 'Social Media Agent'
-    url: 'http://www.brandwatch.com/magpie-crawler/'
-    producer:
-      name: 'Brandwatch'
-      url: 'http://www.brandwatch.com'
--
-  user_agent: MagpieRSS/0.72 (+http://magpierss.sf.net)
-  bot:
-    name: 'MagpieRSS'
-    url: 'http://magpierss.sourceforge.net/'
-    category: 'Feed Parser'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)
-  bot:
-    name: 'Meanpath Bot'
-    category: 'Search bot'
-    url: 'http://www.meanpath.com/meanpathbot.html'
-    producer:
-      name: 'Meanpath'
-      url: 'http://www.meanpath.com'
--
-  user_agent: Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)
-  bot:
-    name: 'Mixrank Bot'
-    category: 'Crawler'
-    url: 'http://mixrank.com'
-    producer:
-      name: 'Online Media Group, Inc.'
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)
-  bot:
-    name: 'MJ12 Bot'
-    category: 'Search bot'
-    url: 'http://majestic12.co.uk/bot.php'
-    producer:
-      name: 'Majestic-12'
-      url: 'http://majestic12.co.uk'
--
-  user_agent: Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)
-  bot:
-    name: 'MojeekBot'
-    category: 'Search bot'
-    url: 'http://www.mojeek.com/bot.html'
-    producer:
-      name: 'Mojeek Ltd.'
-      url: 'http://www.mojeek.com'
--
-  user_agent: NalezenCzBot/1.0 (http://www.nalezen.cz/about-crawler)
-  bot:
-    name: 'NalezenCzBot'
-    category: 'Crawler'
-    url: 'http://www.nalezen.cz/about-crawler'
-    producer:
-      name: 'Jaroslav Kuboš'
-      url: ''
--
-  user_agent: Mozilla/4.0 (compatible; Netcraft Web Server Survey)
-  bot:
-    name: 'Netcraft Survey Bot'
-    category: 'Search bot'
-    url: ''
-    producer:
-      name: 'Netcraft'
-      url: 'http://www.netcraft.com'
--
-  user_agent: Netvibes (http://www.netvibes.com)
-  bot:
-    name: 'Netvibes'
-    url: 'http://www.netvibes.com/'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: 'Netvibes (http://www.netvibes.com/; 8 subscribers; feedID: 2244192)'
-  bot:
-    name: 'Netvibes'
-    url: 'http://www.netvibes.com/'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: NewsBlur Feed Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
-  bot:
-    name: 'NewsBlur'
-    url: 'http://www.newsblur.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: NewsBlur Feed Finder (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
-  bot:
-    name: 'NewsBlur'
-    url: 'http://www.newsblur.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: NewsBlur Page Fetcher - 7 subscribers - http://www.newsblur.com/site/3966817/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
-  bot:
-    name: 'NewsBlur'
-    url: 'http://www.newsblur.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: NewsBlur Favicon Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
-  bot:
-    name: 'NewsBlur'
-    url: 'http://www.newsblur.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: NewsGatorOnline/2.0 (http://www.newsgator.com; 2 subscribers)
-  bot:
-    name: NewsGator
-    url: 'http://www.newsgator.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: nlcrawler/1.0 (+http://northernlight.com/)
-  bot:
-    name: 'NLCrawler'
-    category: 'Crawler'
-    url: ''
-    producer:
-      name: 'Northern Light'
-      url: 'http://northernlight.com'
--
-  user_agent: omgilibot/0.3 +http://www.omgili.com/Crawler.html
-  bot:
-    name: 'Omgili bot'
-    category: 'Search bot'
-    url: 'http://www.omgili.com/Crawler.html'
-    producer:
-      name: 'Omgili'
-      url: 'http://www.omgili.com'
--
-  user_agent: Mozilla/5.0 (compatible; OpenindexSpider; +http://www.openindex.io/en/webmasters/spider.html)
-  bot:
-    name: 'Openindex Spider'
-    category: 'Search bot'
-    url: 'http://www.openindex.io/en/webmasters/spider.html'
-    producer:
-      name: 'Openindex B.V.'
-      url: 'http://www.openindex.io'
--
-  user_agent: Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )
-  bot:
-    name: 'OpenLinkProfiler'
-    category: 'Crawler'
-    url: 'http://openlinkprofiler.org/bot'
-    producer:
-      name: 'Axandra GmbH'
-      url: 'http://www.axandra.com'
--
-  user_agent: OpenWebSpider v0.1.4 (http://www.openwebspider.org/)
-  bot:
-    name: 'OpenWebSpider'
-    category: 'Crawler'
-    url: 'http://www.openwebspider.org'
-    producer:
-      name: 'OpenWebSpider Lab'
-      url: 'http://lab.openwebspider.org'
--
-  user_agent: Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)
-  bot:
-    name: 'PaperLiBot'
-    category: 'Search bot'
-    url: 'http://support.paper.li/entries/20023257-what-is-paper-li'
-    producer:
-      name: 'Smallrivers SA'
-      url: 'http://www.paper.li'
--
-  user_agent: psbot/0.1 (+http://www.picsearch.com/bot.html)
-  bot:
-    name: 'Picsearch bot'
-    category: 'Search bot'
-    url: 'http://www.picsearch.com/bot.html'
-    producer:
-      name: 'Picsearch'
-      url: 'http://www.picsearch.com'
--
-  user_agent: psbot-page (+http://www.picsearch.com/bot.html)
-  bot:
-    name: 'Picsearch bot'
-    category: 'Search bot'
-    url: 'http://www.picsearch.com/bot.html'
-    producer:
-      name: 'Picsearch'
-      url: 'http://www.picsearch.com'
--
-  user_agent: Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
-  bot:
-    name: 'Pingdom Bot'
-    category: 'Site Monitors'
-    url: ''
-    producer:
-      name: 'Pingdom AB'
-      url: 'https://www.pingdom.com'
--
-  user_agent: QuerySeekerSpider ( http://queryseeker.com/bot.html )
-  bot:
-    name: 'QuerySeekerSpider'
-    category: 'Crawler'
-    url: 'http://queryseeker.com/bot.html'
-    producer:
-      name: 'QueryEye Inc.'
-      url: 'http://queryeye.com'
--
-  user_agent: Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)
-  bot:
-    name: 'Reddit Bot'
-    category: 'Social Media Agent'
-    url: 'http://www.reddit.com/feedback'
-    producer:
-      name: 'reddit inc.'
-      url: 'http://www.reddit.com'
--
-  user_agent: Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)
-  bot:
-    name: 'Rogerbot'
-    category: 'Crawler'
-    url: 'http://moz.com/help/pro/what-is-rogerbot-'
-    producer:
-      name: 'SEOmoz, Inc.'
-      url: 'http://moz.com/'
--
-  user_agent: rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)
-  bot:
-    name: 'Rogerbot'
-    category: 'Crawler'
-    url: 'http://moz.com/help/pro/what-is-rogerbot-'
-    producer:
-      name: 'SEOmoz, Inc.'
-      url: 'http://moz.com/'
--
-  user_agent: rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)
-  bot:
-    name: 'Rogerbot'
-    category: 'Crawler'
-    url: 'http://moz.com/help/pro/what-is-rogerbot-'
-    producer:
-      name: 'SEOmoz, Inc.'
-      url: 'http://moz.com/'
--
-  user_agent: Screaming Frog SEO Spider/2.22
-  bot:
-    name: 'Screaming Frog SEO Spider'
-    category: 'Crawler'
-    url: 'http://www.screamingfrog.co.uk/seo-spider'
-    producer:
-      name: 'Screaming Frog Ltd'
-      url: 'http://www.screamingfrog.co.uk'
--
-  user_agent: ScreenerBot Crawler Beta 2.0 (+http://www.ScreenerBot.com)
-  bot:
-    name: 'ScreenerBot'
-    category: 'Crawler'
-    url: 'http://www.screenerbot.com'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; SemrushBot/0.97; +http://www.semrush.com/bot.html)
-  bot:
-    name: 'Semrush Bot'
-    category: 'Crawler'
-    url: 'http://www.semrush.com/bot.html'
-    producer:
-      name: 'SEMrush'
-      url: 'http://www.semrush.com'
--
-  user_agent: SensikaBot/x.33 (+http://sensika.com)
-  bot:
-    name: 'Sensika Bot'
-    category: ''
-    url: ''
-    producer:
-      name: 'Sensika'
-      url: 'http://sensika.com'
--
-  user_agent: SEOENGWorldBot/1.0 (+http://www.seoengine.com/seoengbot.htm)
-  bot:
-    name: 'SEOENGBot'
-    category: 'Crawler'
-    url: 'http://www.seoengine.com/seoengbot.htm'
-    producer:
-      name: 'SEO Engine'
-      url: 'http://www.seoengine.com'
--
-  user_agent: Mozilla/5.0 (compatible; SeznamBot/3.1-test1; +http://fulltext.sblog.cz/)
-  bot:
-    name: 'Seznam Bot'
-    category: 'Search bot'
-    url: 'http://www.mapy.cz/cz/seznambot.html'
-    producer:
-      name: 'Seznam.cz, a.s.'
-      url: 'http://www.seznam.cz/'
--
-  user_agent: Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://fulltext.sblog.cz/)
-  bot:
-    name: 'Seznam Bot'
-    category: 'Search bot'
-    url: 'http://www.mapy.cz/cz/seznambot.html'
-    producer:
-      name: 'Seznam.cz, a.s.'
-      url: 'http://www.seznam.cz/'
--
-  user_agent: SeznamBot/3.0 (+http://fulltext.sblog.cz/)
-  bot:
-    name: 'Seznam Bot'
-    category: 'Search bot'
-    url: 'http://www.mapy.cz/cz/seznambot.html'
-    producer:
-      name: 'Seznam.cz, a.s.'
-      url: 'http://www.seznam.cz/'
--
-  user_agent: Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)
-  bot:
-    name: 'Seznam Bot'
-    category: 'Search bot'
-    url: 'http://www.mapy.cz/cz/seznambot.html'
-    producer:
-      name: 'Seznam.cz, a.s.'
-      url: 'http://www.seznam.cz/'
--
-  user_agent: ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)
-  bot:
-    name: 'ShopWiki'
-    category: 'Search tools'
-    url: 'http://www.shopwiki.com/wiki/Help:Bot'
-    producer:
-      name: 'ShopWiki Corp.'
-      url: 'http://www.shopwiki.com'
--
-  user_agent: SilverReader/1.0; http://silverreader.com
-  bot:
-    name: 'SilverReader'
-    url: 'http://silverreader.com'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20121030175911
-  bot:
-    name: 'SimplePie'
-    url: 'http://www.simplepie.org'
-    category: 'Feed Parser'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
-  bot:
-    name: 'SimplePie'
-    url: 'http://www.simplepie.org'
-    category: 'Feed Parser'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: SimplePie/1.2.1-dev (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20130514092120
-  bot:
-    name: 'SimplePie'
-    url: 'http://www.simplepie.org'
-    category: 'Feed Parser'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)
-  bot:
-    name: 'SISTRIX Crawler'
-    category: 'Crawler'
-    url: 'http://crawler.sistrix.net'
-    producer:
-      name: 'SISTRIX GmbH'
-      url: 'http://www.sistrix.de'
--
-  user_agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
-  bot:
-    name: 'Sogou Spider'
-    category: 'Search bot'
-    url: 'http://www.sogou.com/docs/help/webmasters.htm'
-    producer:
-      name: 'Sohu, Inc.'
-      url: 'http://www.sogou.com'
--
-  user_agent: Sogou inst spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
-  bot:
-    name: 'Sogou Spider'
-    category: 'Search bot'
-    url: 'http://www.sogou.com/docs/help/webmasters.htm'
-    producer:
-      name: 'Sohu, Inc.'
-      url: 'http://www.sogou.com'
--
-  user_agent: Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
-  bot:
-    name: 'Sogou Spider'
-    category: 'Search bot'
-    url: 'http://www.sogou.com/docs/help/webmasters.htm'
-    producer:
-      name: 'Sohu, Inc.'
-      url: 'http://www.sogou.com'
--
-  user_agent: New-Sogou-Spider/1.0 (compatible; MSIE 5.5; Windows 98)
-  bot:
-    name: 'Sogou Spider'
-    category: 'Search bot'
-    url: 'http://www.sogou.com/docs/help/webmasters.htm'
-    producer:
-      name: 'Sohu, Inc.'
-      url: 'http://www.sogou.com'
--
-  user_agent: Sosospider+(+http://help.soso.com/webspider.htm)
-  bot:
-    name: 'Soso Spider'
-    category: 'Search bot'
-    url: 'http://help.soso.com/webspider.htm'
-    producer:
-      name: 'Tencent Holdings'
-      url: 'http://www.soso.com'
--
-  user_agent: 'Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch!'
-  bot:
-    name: 'Superfeedr Bot'
-    category: 'Feed Fetcher'
-    url: ''
-    producer:
-      name: 'Superfeedr'
-      url: 'https://superfeedr.com/'
--
-  user_agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.19; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/2010040121 Firefox/3.0.19
-  bot:
-    name: 'Spinn3r'
-    category: 'Crawler'
-    url: 'http://spinn3r.com/robot'
-    producer:
-      name: 'Tailrank Inc'
-      url: 'http://spinn3r.com'
--
-  user_agent: Mozilla/5.0 (compatible; SputnikBot/2.2)
-  bot:
-    name: 'Sputnik Bot'
-    category: ''
-    url: ''
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (compatible; SputnikImageBot/2.2)
-  bot:
-    name: 'Sputnik Bot'
-    category: ''
-    url: ''
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)
-  bot:
-    name: 'Survey Bot'
-    category: 'Search bot'
-    url: 'http://www.domaintools.com/webmasters/surveybot.php'
-    producer:
-      name: 'Domain Tools'
-      url: 'http://www.domaintools.com'
--
-  user_agent: TinEye-bot/0.02 (see http://www.tineye.com/crawler.html)
-  bot:
-    name: 'TinEye Crawler'
-    category: 'Search bot'
-    url: 'http://www.tineye.com/crawler.html'
-    producer:
-      name: 'Idée Inc.'
-      url: 'http://ideeinc.com'
--
-  user_agent: 'Tiny Tiny RSS/1.10 (http://tt-rss.org/)'
-  bot:
-    name: 'Tiny Tiny RSS'
-    url: 'http://tt-rss.org'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: 'Tiny Tiny RSS/1.11.4c63934 (http://tt-rss.org/)'
-  bot:
-    name: 'Tiny Tiny RSS'
-    url: 'http://tt-rss.org'
-    category: 'Feed Fetcher'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: TurnitinBot/3.0 (http://www.turnitin.com/robot/crawlerinfo.html)
-  bot:
-    name: 'TurnitinBot'
-    category: 'Crawler'
-    url: 'http://www.turnitin.com/robot/crawlerinfo.html'
-    producer:
-      name: 'iParadigms, LLC.'
-      url: 'http://www.turnitin.com'
--
-  user_agent: TweetedTimes Bot/1.0 (Mozilla/5.0 Compatible, +http://tweetedtimes.com)
-  bot:
-    name: 'TweetedTimes Bot'
-    category: 'Crawler'
-    url: 'http://tweetedtimes.com'
-    producer:
-      name: 'TweetedTimes'
-      url: 'http://tweetedtimes.com/'
--
-  user_agent: Mozilla/5.0 (compatible; TweetedTimes Bot/1.0; +http://tweetedtimes.com)
-  bot:
-    name: 'TweetedTimes Bot'
-    category: 'Crawler'
-    url: 'http://tweetedtimes.com'
-    producer:
-      name: 'TweetedTimes'
-      url: 'http://tweetedtimes.com/'
--
-  user_agent: Mozilla/5.0 (compatible; TweetmemeBot/3.0; +http://tweetmeme.com/)
-  bot:
-    name: 'Tweetmeme Bot'
-    category: 'Crawler'
-    url: 'http://tweetmeme.com/'
-    producer:
-      name: 'Mediasift'
-      url: ''
--
-  user_agent: Twitterbot/1.0
-  bot:
-    name: 'Twitterbot'
-    category: 'Social Media Agent'
-    url: 'https://dev.twitter.com/docs/cards/getting-started'
-    producer:
-      name: 'Twitter'
-      url: 'http://www.twitter.com'
--
-  user_agent: Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)
-  bot:
-    name: 'Uptime Robot'
-    category: 'Site Monitor'
-    url: ''
-    producer:
-      name: 'Uptime Robot'
-      url: 'http://uptimerobot.com'
--
-  user_agent: Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)
-  bot:
-    name: 'URLAppendBot'
-    category: 'Crawler'
-    url: 'http://www.profound.net/urlappendbot.html'
-    producer:
-      name: 'Profound Networks'
-      url: 'http://www.profound.net'
--
-  user_agent: Mozilla/5.0 (compatible; VSMCrawler; http://www.visualsitemapper.com/crawler/)
-  bot:
-    name: 'Visual Site Mapper Crawler'
-    category: 'Crawler'
-    url: 'http://www.visualsitemapper.com/crawler'
-    producer:
-      name: 'Alentum Software Ltd.'
-      url: 'http://www.alentum.com'
--
-  user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1;fr;rv:1.8.1) VoilaBotCollector BETA 0.1 (http://www.voila.com/)
-  bot:
-    name: 'Voila Bot'
-    category: 'Search bot'
-    url: 'http://www.voila.fr'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)
-  bot:
-    name: 'Voila Bot'
-    category: 'Search bot'
-    url: 'http://www.voila.fr'
-    producer:
-      name: ''
-      url: ''
--
-  user_agent: Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0 (See <http://validator.w3.org/services>)
-  bot:
-    name: 'W3C CSS Validator'
-    category: 'Validator'
-    url: 'http://jigsaw.w3.org/css-validator'
-    producer:
-      name: 'W3C'
-      url: 'http://www.w3.org'
--
-  user_agent: W3C_I18n-Checker/1.0 (http://validator.w3.org/services)
-  bot:
-    name: 'W3C I18N Checker'
-    category: 'Validator'
-    url: 'http://validator.w3.org/i18n-checker'
-    producer:
-      name: 'W3C'
-      url: 'http://www.w3.org'
--
-  user_agent: W3C-checklink/4.0 [4.4] libwww-perl/5.803
-  bot:
-    name: 'W3C Link Checker'
-    category: 'Validator'
-    url: 'http://validator.w3.org/checklink'
-    producer:
-      name: 'W3C'
-      url: 'http://www.w3.org'
--
-  user_agent: W3C-checklink/4.81 libwww-perl/5.836
-  bot:
-    name: 'W3C Link Checker'
-    category: 'Validator'
-    url: 'http://validator.w3.org/checklink'
-    producer:
-      name: 'W3C'
-      url: 'http://www.w3.org'
--
-  user_agent: W3C_Validator/1.3 http://validator.w3.org/services
-  bot:
-    name: 'W3C Markup Validation Service'
-    category: 'Validator'
-    url: 'http://validator.w3.org/services'
-    producer:
-      name: 'W3C'
-      url: 'http://www.w3.org'
--
-  user_agent: W3C_Validator/1.767
-  bot:
-    name: 'W3C Markup Validation Service'
-    category: 'Validator'
-    url: 'http://validator.w3.org/services'
-    producer:
-      name: 'W3C'
-      url: 'http://www.w3.org'
--
-  user_agent: W3C-mobileOK/DDC-1.0 (see http://www.w3.org/2006/07/mobileok-ddc)
-  bot:
-    name: 'W3C MobileOK Checker'
-    category: 'Validator'
-    url: 'http://validator.w3.org/mobile'
-    producer:
-      name: 'W3C'
-      url: 'http://www.w3.org'
--
-  user_agent: W3C_Unicorn/1.0 (http://validator.w3.org/services)
-  bot:
-    name: 'W3C Unified Validator'
-    category: 'Validator'
-    url: 'http://validator.w3.org/unicorn'
-    producer:
-      name: 'W3C'
-      url: 'http://www.w3.org'
--
-  user_agent: WebbCrawler 1.0 ( http://badcheese.com/crawler.html )
-  bot:
-    name: 'WebbCrawler'
-    category: 'Crawler'
-    url: 'http://badcheese.com/crawler.html'
-    producer:
-      name: 'Steve Webb'
-      url: 'http://badcheese.com'
--
-  user_agent: WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)
-  bot:
-    name: 'WeSEE:Search'
-    category: 'Search bot'
-    url: 'http://www.wesee.com/bot'
-    producer:
-      name: 'WeSEE Ltd'
-      url: 'http://www.wesee.com'
--
-  user_agent: Wotbox/2.01 (+http://www.wotbox.com/bot/)
-  bot:
-    name: 'Wotbox'
-    category: 'Search bot'
-    url: 'http://www.wotbox.com/bot/'
-    producer:
-      name: 'Wotbox'
-      url: 'http://www.wotbox.com'
--
-  user_agent: yacybot (freeworld/global; x86 Windows XP 5.1; java 1.7.0_21; GMT+04:00/ru) http://yacy.net/bot.html
-  bot:
-    name: 'YaCy'
-    category: 'Search bot'
-    url: 'http://yacy.net/bot.html'
-    producer:
-      name: 'YaCy'
-      url: 'http://yacy.net'
--
-  user_agent: yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
-  bot:
-    name: 'YaCy'
-    category: 'Search bot'
-    url: 'http://yacy.net/bot.html'
-    producer:
-      name: 'YaCy'
-      url: 'http://yacy.net'
--
-  user_agent: Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp) NOT Firefox/3.5
-  bot:
-    name: 'Yahoo! Slurp'
-    category: 'Search bot'
-    url: 'http://help.yahoo.com/ysearch/slurp'
-    producer:
-      name: 'Yahoo! Inc.'
-      url: 'http://www.yahoo.com'
--
-  user_agent: Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
-  bot:
-    name: 'Yahoo! Slurp'
-    category: 'Search bot'
-    url: 'http://help.yahoo.com/ysearch/slurp'
-    producer:
-      name: 'Yahoo! Inc.'
-      url: 'http://www.yahoo.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; B; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; B; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexNewslinks; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Mozilla/5.0 (compatible; Linux i686; Yandex.Gazeta Bot/1.0; +http://gazeta.yandex.ru)
-  bot:
-    name: 'Yandex Bot'
-    category: 'Search bot'
-    url: 'http://www.yandex.com/bots'
-    producer:
-      name: 'Yandex LLC'
-      url: 'http://company.yandex.com'
--
-  user_agent: Yeti/1.1 (Naver Corp.; http://help.naver.com/robots/)
-  bot:
-    name: 'Yeti/Naverbot'
-    category: 'Search bot'
-    url: 'http://help.naver.com/robots/'
-    producer:
-      name: 'Naver'
-      url: 'http://www.naver.com'
--
-  user_agent: Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )
-  bot:
-    name: 'Youdao Bot'
-    category: 'Search bot'
-    url: 'http://www.youdao.com/help/webmaster/spider'
-    producer:
-      name: 'NetEase, Inc.'
-      url: 'http://corp.163.com'
--
-  user_agent: Mozilla/5.0 (compatible; YRSpider; +http://www.yunyun.com/SiteInfo.php?r=about)
-  bot:
-    name: 'Yunyun Bot'
-    category: 'Search bot'
-    url: 'http://www.yunyun.com/SiteInfo.php?r=about'
-    producer:
-      name: 'YunYun'
-      url: 'http://www.yunyun.com'
--
-  user_agent: Mozilla/5.0 (compatible; YRSpider; +http://www.yunrang.com/yrspider.html)
-  bot:
-    name: 'Yunyun Bot'
-    category: 'Search bot'
-    url: 'http://www.yunyun.com/SiteInfo.php?r=about'
-    producer:
-      name: 'YunYun'
-      url: 'http://www.yunyun.com'
--
-  user_agent: Mozilla/5.0 (compatible; YYSpider; +http://www.yunyun.com/spider.html)
-  bot:
-    name: 'Yunyun Bot'
-    category: 'Search bot'
-    url: 'http://www.yunyun.com/SiteInfo.php?r=about'
-    producer:
-      name: 'YunYun'
-      url: 'http://www.yunyun.com'
--
-  user_agent: Zookabot/2.5;++http://zookabot.com
-  bot:
-    name: 'Zookabot'
-    category: 'Crawler'
-    url: 'http://zookabot.com'
-    producer:
-      name: 'Hwacha ApS'
-      url: 'http://hwacha.dk'
--
-  user_agent: Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)
-  bot:
-    name: 'ZumBot'
-    category: 'Search bot'
-    url: 'http://help.zum.com/inquiry'
-    producer:
-      name: 'ZUM internet'
-      url: 'http://www.zuminternet.com/'
--
-  user_agent: ZumBot/1.0 (ZUM Search; http://help.zum.com/inquiry)
-  bot:
-    name: 'ZumBot'
-    category: 'Search bot'
-    url: 'http://help.zum.com/inquiry'
-    producer:
-      name: 'ZUM internet'
-      url: 'http://www.zuminternet.com/'
--
-  user_agent: Your Nutch Spider/Nutch-2.2.1
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: Nutch12/Nutch-1.2
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: spider/Nutch-1.5.1 (spider; http://www.xxx.com)
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: MySpider/Nutch-2.2
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: My Nutch Spider/Nutch-1.6
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: NutchSpider/Nutch-1.4
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: testnutch/Nutch-1.8
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: Spider/Nutch-2.3-SNAPSHOT (Webcrawler)
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: your sipder name/Nutch-1.7
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: 5acd734a-1ed3-4a17-ad9d-0b09ae7bc2d2/Nutch-1.7
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: Mozilla/5.0 (compatible; summers;)/Nutch-1.7
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8/Nutch-1.7
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: CB/Nutch-1.7
-  bot:
-    name: 'Nutch-based Bot'
-    category: 'crawler'
-    url: 'https://nutch.apache.org'
-    producer:
-      name: 'The Apache Software Foundation'
-      url: 'http://www.apache.org/foundation/'
--
-  user_agent: NetLyzer FastProbe
-  bot:
-    name: 'NetLyzer FastProbe'
--
-  user_agent: StatusCake Error Detector Crawler V0.01
-  bot:
-    name: 'Generic Bot'
--
-  user_agent: niki-bot
-  bot:
-    name: 'Generic Bot'