device_detector 1.0.5 → 1.0.6

data/spec/fixtures/detector/bots.yml

@@ -1,45 +1,51 @@
 ---
-- 
+-
   user_agent: 360spider-image
   bot:
     name: 360Spider
     category: Search bot
-    url: http://www.so.com/help/help_3_2.html
+    url: https://www.so.com/help/help_3_2.html
     producer:
       name: Online Media Group, Inc.
       url: ""
-- 
+-
   user_agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MDDR; .NET4.0C; .NET4.0E; .NET CLR 1.1.4322; Tablet PC 2.0); 360Spider
   bot:
     name: 360Spider
     category: Search bot
-    url: http://www.so.com/help/help_3_2.html
+    url: https://www.so.com/help/help_3_2.html
     producer:
       name: Online Media Group, Inc.
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider
   bot:
     name: 360Spider
     category: Search bot
-    url: http://www.so.com/help/help_3_2.html
+    url: https://www.so.com/help/help_3_2.html
     producer:
       name: Online Media Group, Inc.
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider
   bot:
     name: 360Spider
     category: Search bot
-    url: http://www.so.com/help/help_3_2.html
+    url: https://www.so.com/help/help_3_2.html
     producer:
       name: Online Media Group, Inc.
       url: ""
-- 
+-
   user_agent: 'ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com'
   bot:
     name: ADMantX
-- 
+-
+  user_agent: ias-va/3.1 (+https://www.admantx.com/service-fetcher.html)
+  bot:
+    name: ADmantX Service Fetcher
+    category: Service bot
+    url: https://www.admantx.com/service-fetcher.html
+-
   user_agent: Aboundex/0.3 (http://www.aboundex.com/crawler/)
   bot:
     name: Aboundexbot
@@ -48,7 +54,7 @@
     producer:
       name: Aboundex.com
       url: http://www.aboundex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; AcoonBot/4.11.1; +http://www.acoon.de/robot.asp)
   bot:
     name: Acoon
@@ -57,7 +63,7 @@
     producer:
       name: Acoon GmbH
       url: http://www.acoon.de
-- 
+-
   user_agent: AddThis.com robot tech.support@clearspring.com
   bot:
     name: AddThis.com
@@ -66,25 +72,34 @@
     producer:
       name: Clearspring Technologies, Inc.
       url: http://www.clearspring.com
-- 
+-
   user_agent: ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)
   bot:
     name: Alexa Crawler
     category: Search bot
-    url: https://alexa.zendesk.com/hc/en-us/sections/200100794-Crawlers
+    url: https://support.alexa.com/hc/en-us/sections/200100794-Crawlers
     producer:
       name: Alexa Internet
-      url: http://www.alexa.com
-- 
+      url: https://www.alexa.com
+-
   user_agent: Mozilla/5.0 (compatible; alexa site audit/1.0; http://www.alexa.com/help/webmasters; )
   bot:
     name: Alexa Site Audit
     category: Site Monitor
-    url: http://www.alexa.com/help/webmasters
+    url: https://support.alexa.com/hc/en-us/articles/200450194
     producer:
       name: Alexa Internet
-      url: http://www.alexa.com
-- 
+      url: https://www.alexa.com
+-
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)
+  bot:
+    name: Amazon Bot
+    category: Crawler
+    url: https://developer.amazon.com/support/amazonbot
+    producer:
+      name: Amazon.com, Inc.
+      url: https://www.amazon.com/
+-
   user_agent: Amazon Route 53 Health Check Service; ref:xxxxxxxx-xxxx-xxxxxxxxx-xxxxxxxxxxxx; report http://amzn.to/xxxxxxx
   bot:
     name: Amazon Route53 Health Check
@@ -92,7 +107,7 @@
     producer:
       name: Amazon Web Services
       url: https://aws.amazon.com/
-- 
+-
   user_agent: Amazon-Route53-Health-Check-Service (ref b0eb04d5-cb5e-40e7-839b-558e52fc3f0d; report http://amzn.to/1vsZADi)
   bot:
     name: Amazon Route53 Health Check
@@ -100,7 +115,7 @@
     producer:
       name: Amazon Web Services
       url: https://aws.amazon.com/
-- 
+-
   user_agent: AmorankSpider/0.1; +http://amorank.com/webcrawler.html
   bot:
     name: Amorank Spider
@@ -109,7 +124,7 @@
     producer:
       name: Amorank
       url: http://www.amorank.com
-- 
+-
   user_agent: 'Curious George - www.analyticsseo.com/crawler'
   bot:
     name: Analytics SEO Crawler
@@ -118,7 +133,7 @@
     producer:
       name: Analytics SEO
       url: http://www.analyticsseo.com
-- 
+-
   user_agent: ApacheBench/2.3
   bot:
     name: ApacheBench
@@ -126,35 +141,44 @@
     url: https://httpd.apache.org/docs/2.4/programs/ab.html
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10 _1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)
   bot:
     name: Applebot
     category: Crawler
-    url: http://www.apple.com/go/applebot
+    url: https://support.apple.com/en-us/HT204683
     producer:
       name: Apple Inc
-      url: http://www.apple.com
-- 
+      url: https://www.apple.com
+-
+  user_agent: AppSignalBot/1.0 (+https://appsignal.com)
+  bot:
+    name: AppSignalBot
+    category: Site Monitor
+    url: https://docs.appsignal.com/uptime-monitoring/
+    producer:
+      name: AppSignal
+      url: https://appsignal.com/
+-
   user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)
   bot:
     name: Applebot
     category: Crawler
-    url: http://www.apple.com/go/applebot
+    url: https://support.apple.com/en-us/HT204683
     producer:
       name: Apple Inc
-      url: http://www.apple.com
-- 
+      url: https://www.apple.com
+-
   user_agent: Arachni/v1.5.1
   bot:
     name: Arachni
     category: Security Checker
-    url: http://www.arachni-scanner.com
+    url: https://www.arachni-scanner.com/
     producer:
       name: Sarosys LLC
-      url: http://www.sarosys.com/
-- 
+      url: https://www.sarosys.com/
+-
   user_agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
   bot:
     name: Ask Jeeves
@@ -163,7 +187,7 @@
     producer:
       name: Ask Jeeves Inc.
       url: http://www.ask.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible;AspiegelBot)
   bot:
     name: AspiegelBot
@@ -172,7 +196,7 @@
     producer:
       name: Huawei
       url: https://www.huawei.com/
-- 
+-
   user_agent: Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)
   bot:
     name: AspiegelBot
@@ -181,7 +205,7 @@
     producer:
       name: Huawei
       url: https://www.huawei.com/
-- 
+-
   user_agent: AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)
   bot:
     name: Awario
@@ -190,7 +214,7 @@
     producer:
       name: Awario
       url: https://awario.com/
-- 
+-
   user_agent: AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)
   bot:
     name: Awario
@@ -199,7 +223,7 @@
     producer:
       name: Awario
       url: https://awario.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
   bot:
     name: BLEXBot Crawler
@@ -208,7 +232,7 @@
     producer:
       name: WebMeUp
       url: http://webmeup.com
-- 
+-
   user_agent: BUbiNG (+http://law.di.unimi.it/BUbiNG.html)
   bot:
     name: BUbiNG
@@ -217,7 +241,7 @@
     producer:
       name: The Laboratory for Web Algorithmics (LAW)
       url: 'http://law.di.unimi.it/software.php#buging'
-- 
+-
   user_agent: Backlink-Check.de (+http://www.backlink-check.de/bot.html)
   bot:
     name: Backlink-Check.de
@@ -226,7 +250,7 @@
     producer:
       name: Mediagreen Medienservice
       url: http://www.backlink-check.de
-- 
+-
   user_agent: BacklinkCrawler (http://www.backlinktest.com/crawler.html)
   bot:
     name: BacklinkCrawler
@@ -235,7 +259,7 @@
     producer:
       name: 2.0Promotion GbR
       url: http://www.backlinktest.com
-- 
+-
   user_agent: Baiduspider+(+http://www.baidu.com/search/spider.htm)
   bot:
     name: Baidu Spider
@@ -244,7 +268,7 @@
     producer:
       name: Baidu
       url: http://www.baidu.com
-- 
+-
   user_agent: Baiduspider-image+(+http://www.baidu.com/search/spider.htm)
   bot:
     name: Baidu Spider
@@ -253,7 +277,7 @@
     producer:
       name: Baidu
       url: http://www.baidu.com
-- 
+-
   user_agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; baidu Transcoder;)
   bot:
     name: Baidu Spider
@@ -262,7 +286,7 @@
     producer:
       name: Baidu
       url: http://www.baidu.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search
   bot:
     name: Baidu Spider
@@ -271,7 +295,7 @@
     producer:
       name: Baidu
       url: http://www.baidu.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
   bot:
     name: Baidu Spider
@@ -280,7 +304,7 @@
     producer:
       name: Baidu
       url: http://www.baidu.com
-- 
+-
   user_agent: Mozilla/5.0 (Linux;u;Android 2.3.7;zh-cn;) AppleWebKit/533.1 (KHTML,like Gecko) Version/4.0 Mobile Safari/533.1 (compatible; +http://www.baidu.com/search/spider.html)
   bot:
     name: Baidu Spider
@@ -289,7 +313,7 @@
     producer:
       name: Baidu
       url: http://www.baidu.com
-- 
+-
   user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8;baidu Transcoder) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729)
   bot:
     name: Baidu Spider
@@ -298,7 +322,13 @@
     producer:
       name: Baidu
       url: http://www.baidu.com
-- 
+-
+  user_agent: Barkrowler/0.9 (+http://www.exensa.com/crawl)
+  bot:
+    name: Barkrowler
+    category: Crawler
+    url: http://www.exensa.com/crawl
+-
   user_agent: Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 2 subscribers)
   bot:
     name: BazQux Reader
@@ -307,7 +337,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
   bot:
     name: BingBot
@@ -316,7 +346,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b
   bot:
     name: BingBot
@@ -325,7 +355,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)
   bot:
     name: BingBot
@@ -334,7 +364,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)
   bot:
     name: BingBot
@@ -343,7 +373,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: msnbot/1.1 (+http://search.msn.com/msnbot.htm)
   bot:
     name: BingBot
@@ -352,7 +382,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: msnbot/2.0b (+http://search.msn.com/msnbot.htm)
   bot:
     name: BingBot
@@ -361,7 +391,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: msnbot/2.0b v1394720947.6703
   bot:
     name: BingBot
@@ -370,7 +400,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: MSNBot/Nutch-1.5.1
   bot:
     name: BingBot
@@ -379,7 +409,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: bitlybot/3.0
   bot:
     name: BitlyBot
@@ -388,7 +418,7 @@
     producer:
       name: Bitly, Inc.
       url: https://bitly.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)
   bot:
     name: Blekkobot
@@ -397,7 +427,7 @@
     producer:
       name: Blekko
       url: http://blekko.com
-- 
+-
   user_agent: Bloglovin/1.0 (http://www.bloglovin.com; 1 subscribers)
   bot:
     name: Bloglovin
@@ -406,7 +436,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Blogtrottr/2.0
   bot:
     name: Blogtrottr
@@ -415,24 +445,24 @@
     producer:
       name: Blogtrottr Ltd
       url: https://blogtrottr.com/
-- 
+-
   user_agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com
   bot:
     name: BoardReader
     category: Search bot
-    url: http://boardreader.com/
+    url: https://boardreader.com/
     producer:
       name: Effyis Inc
-      url: http://boardreader.com/
-- 
+      url: https://boardreader.com/
+-
   user_agent: BoardReader Blog Indexer(http://boardreader.com)
   bot:
     name: BoardReader Blog Indexer
     category: Crawler
     producer:
       name: BoardReader
-      url: http://boardreader.com/
-- 
+      url: https://boardreader.com/
+-
   user_agent: Mozilla/5.0 (compatible; BountiiBot/1.1; +http://bountii.com/contact.php)
   bot:
     name: Bountii Bot
@@ -441,7 +471,7 @@
     producer:
       name: Bountii Inc.
       url: http://bountii.com
-- 
+-
   user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)
   bot:
     name: BrandVerity
@@ -450,7 +480,7 @@
     producer:
       name: BrandVerity, Inc.
       url: https://www.brandverity.com/
-- 
+-
   user_agent: Browsershots
   bot:
     name: Browsershots
@@ -459,7 +489,7 @@
     producer:
       name: Browsershots.org
       url: http://browsershots.org
-- 
+-
   user_agent: Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)
   bot:
     name: Buck
@@ -468,7 +498,7 @@
     producer:
       name: Hypefactors A/S
       url: https://hypefactors.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
   bot:
     name: Butterfly Robot
@@ -477,7 +507,7 @@
     producer:
       name: Topsy Labs
       url: http://labs.topsy.com
-- 
+-
   user_agent: Mozilla/5.0 (Macintosh; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
   bot:
     name: Butterfly Robot
@@ -486,7 +516,7 @@
     producer:
       name: Topsy Labs
       url: http://labs.topsy.com
-- 
+-
   user_agent: Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.5668.1446 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com
   bot:
     name: Bytespider
@@ -495,7 +525,7 @@
     producer:
       name: ByteDance Ltd.
       url: https://bytedance.com/
-- 
+-
   user_agent: CATExplorador/1.0beta (sistemes at domini dot cat; http://domini.cat/catexplorador.html)
   bot:
     name: CATExplorador
@@ -504,7 +534,7 @@
     producer:
       name: Fundació puntCAT
       url: https://fundacio.cat/ca/domini/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; CareerBot/1.1; +http://www.career-x.de/bot.html)
   bot:
     name: CareerBot
@@ -513,7 +543,7 @@
     producer:
       name: career-x GmbH
       url: http://www.career-x.de
-- 
+-
   user_agent: Castro 2, Episode Duration Lookup
   bot:
     name: Castro 2
@@ -522,7 +552,7 @@
     producer:
       name: Supertop
       url: http://supertop.co
-- 
+-
   user_agent: Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)
   bot:
     name: Cliqzbot
@@ -531,7 +561,7 @@
     producer:
       name: 10betterpages GmbH
       url: http://cliqz.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Cloudflare-AMP/1.0; +https://amp.cloudflare.com/doc/fetcher.html) AppleWebKit/534.34
   bot:
     name: CloudFlare AMP Fetcher
@@ -540,7 +570,7 @@
     producer:
       name: CloudFlare
       url: http://www.cloudflare.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34
   bot:
     name: CloudFlare Always Online
@@ -549,7 +579,7 @@
     producer:
       name: CloudFlare
       url: http://www.cloudflare.com
-- 
+-
   user_agent: collectd/5.5.1
   bot:
     name: Collectd
@@ -558,7 +588,7 @@
     producer:
       name: Collectd
       url: https://collectd.org/
-- 
+-
   user_agent: CommaFeed/1.0 (http://www.commafeed.com)
   bot:
     name: CommaFeed
@@ -567,79 +597,79 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: coccocbot-web/1.0 (+http://help.coccoc.com/searchengine)
   bot:
     name: Cốc Cốc Bot
-    category: Search bot
     url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    category: Search bot
     producer:
       name: Cốc Cốc
       url: https://coccoc.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)
   bot:
     name: Cốc Cốc Bot
-    category: Search bot
     url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    category: Search bot
     producer:
       name: Cốc Cốc
       url: https://coccoc.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; coccocbot-ads/1.0; +http://help.coccoc.com/searchengine)
   bot:
     name: Cốc Cốc Bot
-    category: Search bot
     url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    category: Search bot
     producer:
       name: Cốc Cốc
       url: https://coccoc.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; coccocbot-fast/1.0; +http://help.coccoc.com/searchengine)
   bot:
     name: Cốc Cốc Bot
-    category: Search bot
     url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    category: Search bot
     producer:
       name: Cốc Cốc
       url: https://coccoc.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)
   bot:
     name: Cốc Cốc Bot
-    category: Search bot
     url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    category: Search bot
     producer:
       name: Cốc Cốc
       url: https://coccoc.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; coccocbot-shopping/1.0; +http://help.coccoc.com/searchengine)
   bot:
     name: Cốc Cốc Bot
-    category: Search bot
     url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    category: Search bot
     producer:
       name: Cốc Cốc
       url: https://coccoc.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)
   bot:
     name: Cốc Cốc Bot
-    category: Search bot
     url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    category: Search bot
     producer:
       name: Cốc Cốc
       url: https://coccoc.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; coccocbot/1.0; +http://help.coccoc.com/searchengine)
   bot:
     name: Cốc Cốc Bot
-    category: Search bot
     url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    category: Search bot
     producer:
       name: Cốc Cốc
       url: https://coccoc.com/
-- 
+-
   user_agent: Datadog Agent/5.10.1
   bot:
     name: Datadog Agent
@@ -648,7 +678,7 @@
     producer:
       name: Datadog
       url: https://www.datadoghq.com/
-- 
+-
   user_agent: Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36
   bot:
     name: Datanyze
@@ -657,7 +687,7 @@
     producer:
       name: Datanyze
       url: https://www.datanyze.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Dataprovider/6.92; +https://www.dataprovider.com/)
   bot:
     name: Dataprovider
@@ -666,7 +696,7 @@
     producer:
       name: Dataprovider B.V.
       url: https://www.dataprovider.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0;
   bot:
     name: Daum
@@ -675,7 +705,7 @@
     producer:
       name: Daum Communications Corp.
       url: http://www.kakaocorp.com/main
-- 
+-
   user_agent: Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1
   bot:
     name: Daum
@@ -684,7 +714,7 @@
     producer:
       name: Daum Communications Corp.
       url: http://www.kakaocorp.com/main
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Dazoobot/0.1; +http://dazoo.fr)
   bot:
     name: Dazoobot
@@ -693,7 +723,7 @@
     producer:
       name: DAZOO.FR
       url: http://dazoo.fr
-- 
+-
   user_agent: Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)
   bot:
     name: Discobot
@@ -702,7 +732,7 @@
     producer:
       name: Discovery Engine
       url: http://discoveryengine.com
-- 
+-
   user_agent: 'Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com'
   bot:
     name: Domain Re-Animator Bot
@@ -711,7 +741,7 @@
     producer:
       name: Domain Re-Animator, LLC
       url: http://domainreanimator.com
-- 
+-
   user_agent: support@domainreanimator.com
   bot:
     name: Domain Re-Animator Bot
@@ -720,7 +750,13 @@
     producer:
       name: Domain Re-Animator, LLC
       url: http://domainreanimator.com
-- 
+-
+  user_agent: Mozilla/5.0 (compatible; Domains Project/1.1.0; +https://domainsproject.org)
+  bot:
+    name: Domains Project
+    category: Crawler
+    url: https://domainsproject.org
+-
   user_agent: dotbot
   bot:
     name: DotBot
@@ -729,7 +765,7 @@
     producer:
       name: SEOmoz, Inc.
       url: http://moz.com/
-- 
+-
   user_agent: DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)
   bot:
     name: DuckDuckGo Bot
@@ -738,7 +774,7 @@
     producer:
       name: DuckDuckGo
       url: https://duckduckgo.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)
   bot:
     name: DuckDuckGo Bot
@@ -747,7 +783,7 @@
     producer:
       name: DuckDuckGo
       url: https://duckduckgo.com/
-- 
+-
   user_agent: EMail Exractor
   bot:
     name: EMail Exractor
@@ -756,7 +792,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)
   bot:
     name: Easou Spider
@@ -765,7 +801,7 @@
     producer:
       name: easou ICP
       url: http://www.easou.com
-- 
+-
   user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us; EasouSpider; +http://www.easou.com/search/spider.html)
   bot:
     name: Easou Spider
@@ -774,7 +810,7 @@
     producer:
       name: easou ICP
       url: http://www.easou.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)
   bot:
     name: Embedly
@@ -783,7 +819,7 @@
     producer:
       name: A Medium, Corp.
       url: https://medium.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)
   bot:
     name: ExaBot
@@ -792,7 +828,7 @@
     producer:
       name: Dassault Systèmes
       url: http://www.3ds.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)
   bot:
     name: ExaBot
@@ -801,7 +837,7 @@
     producer:
       name: Dassault Systèmes
       url: http://www.3ds.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)
   bot:
     name: ExaBot
@@ -810,7 +846,7 @@
     producer:
       name: Dassault Systèmes
       url: http://www.3ds.com
-- 
+-
   user_agent: ExactSeek Crawler (nutch 1.4)/Nutch-1.4 (ExactSeek Crawler; http://www.exactseek.com)
   bot:
     name: ExactSeek Crawler
@@ -819,7 +855,7 @@
     producer:
       name: Jayde Online, Inc.
       url: http://www.jaydeonlineinc.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Ezooms/1.0; help@moz.com)
   bot:
     name: Ezooms
@@ -828,7 +864,7 @@
     producer:
       name: SEOmoz, Inc.
       url: http://moz.com/
-- 
+-
   user_agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
   bot:
     name: Facebook External Hit
@@ -837,7 +873,7 @@
     producer:
       name: Facebook
       url: http://www.facebook.com
-- 
+-
   user_agent: facebookexternalua
   bot:
     name: Facebook External Hit
@@ -846,7 +882,7 @@
     producer:
       name: Facebook
       url: http://www.facebook.com
-- 
+-
   user_agent: facebookplatform/1.0 (+http://developers.facebook.com)
   bot:
     name: Facebook External Hit
@@ -855,7 +891,7 @@
     producer:
       name: Facebook
       url: http://www.facebook.com
-- 
+-
   user_agent: Feed Wrangler/1.0 (3 subscribers; feed-id=248559; http://feedwrangler.net; Allow like Gecko)
   bot:
     name: Feed Wrangler
@@ -864,7 +900,7 @@
     producer:
       name: 'David Smith & Developing Perspective, LLC'
       url: https://david-smith.org
-- 
+-
   user_agent: FeedBurner/1.0 (http://www.FeedBurner.com)
   bot:
     name: FeedBurner
@@ -873,7 +909,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: 'Feedbin - 9 subscribers'
   bot:
     name: Feedbin
@@ -882,7 +918,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)
   bot:
     name: Feedly
@@ -891,7 +927,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: FeedlyApp/1.0 (http://www.feedly.com)
   bot:
     name: Feedly
@@ -900,7 +936,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: FeedlyBot/1.0 (http://feedly.com)
   bot:
     name: Feedly
@@ -909,7 +945,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: MetaFeedly/1.0 (http://www.feedly.com)
   bot:
     name: Feedly
@@ -918,7 +954,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Feedspot http://www.feedspot.com
   bot:
     name: Feedspot
@@ -927,7 +963,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)
   bot:
     name: Fever
@@ -936,13 +972,13 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)
   bot:
     name: Findxbot
     category: Crawler
     url: http://www.findxbot.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)
   bot:
     name: Flipboard
@@ -951,7 +987,7 @@
     producer:
       name: Flipboard
       url: http://flipboard.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; FlipboardRSS/1.2; +http://flipboard.com/browserproxy)
   bot:
     name: Flipboard
@@ -960,19 +996,19 @@
     producer:
       name: Flipboard
       url: http://flipboard.com/
-- 
+-
   user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
   bot:
     name: FreshRSS
     category: Feed Fetcher
     url: https://freshrss.org/
-- 
+-
   user_agent: FreshRSS/1.12.0 (Linux; https://freshrss.org)
   bot:
     name: FreshRSS
     category: Feed Fetcher
     url: https://freshrss.org/
-- 
+-
   user_agent: Mozilla/5.0 (X11; Linux x86_64; GTmetrix https://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36
   bot:
     name: GTmetrix
@@ -981,55 +1017,55 @@
     producer:
       name: Carbon60 Operating Co. Ltd.
       url: https://www.carbon60.com/
-- 
+-
   user_agent: 7Siters/1.07 (+https://7ooo.ru/siters/)
   bot:
     name: Generic Bot
-- 
+-
   user_agent: COMODO SSL Checker
   bot:
     name: Generic Bot
-- 
+-
   user_agent: help@dataminr.com
   bot:
     name: Generic Bot
-- 
+-
   user_agent: 'HybridBot (hybrid.ru/about. If our bot caused problems please contact us. Contact email: m.lyashkov@targetix.net)'
   bot:
     name: Generic Bot
-- 
+-
   user_agent: 'Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)'
   bot:
     name: Generic Bot
-- 
+-
   user_agent: Mozilla/5.0 (compatible; +http://tweetedtimes.com)
   bot:
     name: Generic Bot
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Shareaholicbot/2.0; +http://www.shareaholic.com/bot)
   bot:
     name: Generic Bot
-- 
+-
   user_agent: Mozilla/5.0 (compatible; TrendsmapResolver/0.1)
   bot:
     name: Generic Bot
-- 
+-
   user_agent: niki-bot
   bot:
     name: Generic Bot
-- 
+-
   user_agent: PHPCrawl
   bot:
     name: Generic Bot
-- 
+-
   user_agent: robots
   bot:
     name: Generic Bot
-- 
+-
   user_agent: SeopultContentAnalyzer/1.0
   bot:
     name: Generic Bot
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)
   bot:
     name: Genieo Web filter
@@ -1038,7 +1074,7 @@
     producer:
       name: Genieo
       url: http://www.genieo.com
-- 
+-
   user_agent: GigablastOpenSource/1.0
   bot:
     name: Gigablast
@@ -1047,7 +1083,7 @@
     producer:
       name: Matt Wells
       url: http://www.gigablast.com/faq.html
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)
   bot:
     name: Gluten Free Crawler
@@ -1056,7 +1092,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)
   bot:
     name: Gmail Image Proxy
@@ -1065,7 +1101,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)
   bot:
     name: Goo
@@ -1074,12 +1110,12 @@
     producer:
       name: NTT Resonant
       url: http://goo.ne.jp
-- 
+-
   user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon
   bot:
     name: Google Favicon
     category: Crawler
-- 
+-
   user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0_1 like Mac OS X; en-us) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Version/4.0.5 Mobile/8A306 Safari/6531.22.7
   bot:
     name: Google PageSpeed Insights
@@ -1088,7 +1124,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1 google_partner_monitoring FWSzVTDDBz14547302713138T
   bot:
     name: Google Partner Monitoring
@@ -1097,7 +1133,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Search Console) Chrome/41.0.2272.118 Safari/537.36
   bot:
     name: Google Search Console
@@ -1106,7 +1142,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)
   bot:
     name: Google Structured Data Testing Tool
@@ -1115,7 +1151,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: AdsBot-Google (+http://www.google.com/adsbot.html)
   bot:
     name: Googlebot
@@ -1124,7 +1160,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: AdsBot-Google-Mobile (+http://www.google.com/mobile/adsbot.html) Mozilla (iPhone; U; CPU iPhone OS 3 0 like Mac OS X) AppleWebKit (KHTML, like Gecko) Mobile Safari
   bot:
     name: Googlebot
@@ -1133,7 +1169,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)
   bot:
     name: Googlebot
@@ -1142,7 +1178,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
   bot:
     name: Googlebot
@@ -1151,7 +1187,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 19 subscribers; feed-id=13965549748850348809)
   bot:
     name: Googlebot
@@ -1160,7 +1196,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 2 subscribers; feed-id=17860707833818568603)
   bot:
     name: Googlebot
@@ -1169,7 +1205,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 375 subscribers; feed-id=15381863289700640853)
   bot:
     name: Googlebot
@@ -1178,7 +1214,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Google-AdWords-Express
   bot:
     name: Googlebot
@@ -1187,7 +1223,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Google-Adwords-Instant (+http://www.google.com/adsbot.html)
   bot:
     name: Googlebot
@@ -1196,7 +1232,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Google-speakr
   bot:
     name: Googlebot
@@ -1205,7 +1241,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Googlebot (gocrawl v0.4)
   bot:
     name: Googlebot
@@ -1214,7 +1250,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Googlebot-Image/1.0
   bot:
     name: Googlebot
@@ -1223,7 +1259,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Googlebot-News (2.3.3, ruby 1.9.3 (2013-11-22))
   bot:
     name: Googlebot
@@ -1232,7 +1268,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Googlebot-Video/1.0
   bot:
     name: Googlebot
@@ -1241,7 +1277,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Googlebot/2.1 (http://www.googlebot.com/bot.html)
   bot:
     name: Googlebot
@@ -1250,7 +1286,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Googlebot/Nutch-1.7
   bot:
     name: Googlebot
@@ -1259,7 +1295,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: GoogleProducer; (+http://goo.gl/7y4SX)
   bot:
     name: Googlebot
@@ -1268,7 +1304,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mediapartners-Google
   bot:
     name: Googlebot
@@ -1277,7 +1313,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible) Feedfetcher-Google;(+http://www.google.com/feedfetcher.html)
   bot:
     name: Googlebot
@@ -1286,7 +1322,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Google-Youtube-Links)
   bot:
     name: Googlebot
@@ -1295,7 +1331,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
   bot:
     name: Googlebot
@@ -1304,7 +1340,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (en-US) AppleWebKit/537.36 (KHTML, like Gecko; Google-Assess) Chrome/34.0.1847.116 Safari/537.36
   bot:
     name: Googlebot
@@ -1313,7 +1349,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (en-us) AppleWebKit/537.36(KHTML, like Gecko; Google-Adwords-DisplayAds-WebRender;) Chrome/41.0.2272.118Safari/537.36
   bot:
     name: Googlebot
@@ -1322,7 +1358,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
   bot:
     name: Googlebot
@@ -1331,7 +1367,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Google-Publisher-Plugin) Chrome/27.0.1453 Mobile Safari/537.36
   bot:
     name: Googlebot
@@ -1340,7 +1376,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Google-AMPHTML)
   bot:
     name: Googlebot
@@ -1349,7 +1385,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)
   bot:
     name: Googlebot
@@ -1358,7 +1394,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; DuplexWeb-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36
   bot:
     name: Googlebot
@@ -1367,7 +1403,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)
   bot:
     name: Googlebot
@@ -1376,7 +1412,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Safari/537.36 (compatible; Google-HotelAdsVerifier/2.0)
   bot:
     name: Googlebot
@@ -1385,7 +1421,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36
   bot:
     name: Googlebot
@@ -1394,7 +1430,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Shopping-Quality +http://www.google.com/merchants/tos/extend/DE/tos.html)
   bot:
     name: Googlebot
@@ -1403,7 +1439,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36
   bot:
     name: Googlebot
@@ -1412,7 +1448,7 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
   user_agent: SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
   bot:
     name: Googlebot
@@ -1421,7 +1457,13 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
-- 
+-
+  user_agent: Grammarly/1.0 (http://www.grammarly.com)
+  bot:
+    name: Grammarly
+    category: Service bot
+    url: https://www.grammarly.com
+-
   user_agent: Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +https://www.grapeshot.com/crawler/)
   bot:
     name: Grapeshot
@@ -1430,7 +1472,7 @@
     producer:
       name: Grapeshot
       url: https://www.grapeshot.com
-- 
+-
   user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; GrapeshotCrawler/2.0; +https://www.grapeshot.com/crawler/)
   bot:
     name: Grapeshot
@@ -1439,7 +1481,7 @@
     producer:
       name: Grapeshot
       url: https://www.grapeshot.com
-- 
+-
   user_agent: HTTPMon/1.0b (http://www.httpmon.com)
   bot:
     name: HTTPMon
@@ -1448,7 +1490,7 @@
     producer:
       name: towards GmbH
       url: http://www.towards.ch/
-- 
+-
   user_agent: 'flieder - neofonie heritrix/1.14.3 (+http://spider.neofonie.de)'
   bot:
     name: Heritrix
@@ -1456,8 +1498,8 @@
     url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; heritrix/1.14.4 +http://webarhive.nlc.gov.cn)
   bot:
     name: Heritrix
@@ -1465,8 +1507,8 @@
     url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.dla-marbach.de)
   bot:
     name: Heritrix
@@ -1474,8 +1516,8 @@
     url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: mozilla/5.0 (compatible; heritrix/1.3.0 http://archive.crawler.org)
   bot:
     name: Heritrix
@@ -1483,8 +1525,8 @@
     url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.baidu.com)
   bot:
     name: Heritrix
@@ -1492,8 +1534,8 @@
     url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.run4dom.com)
   bot:
     name: Heritrix
@@ -1501,8 +1543,8 @@
     url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20130207.001528 +http://webarchiv.cz/kontakty/)
   bot:
     name: Heritrix
@@ -1510,8 +1552,8 @@
     url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)
   bot:
     name: Heritrix
@@ -1519,8 +1561,8 @@
     url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Heurekabot-Feed/1.0 (+https://sluzby.heureka.cz/napoveda/heurekabot/)
   bot:
     name: Heureka Feed
@@ -1529,16 +1571,16 @@
     producer:
       name: Heureka.cz, a.s.
       url: https://www.heureka.cz/
-- 
+-
   user_agent: HubPages V0.2.2 (http://hubpages.com/help/crawlingpolicy)
   bot:
     name: HubPages
     category: Crawler
-    url: http://hubpages.com/help/crawlingpolicy
+    url: https://hubpages.com/help/crawlingpolicy
     producer:
-      name: HubPages
-      url: http://hubpages.com/
-- 
+      name: HubPages, Inc.
+      url: https://discover.hubpages.com/
+-
   user_agent: HubSpot Website Grader (web-crawlers@hubspot.com)
   bot:
     name: HubSpot
@@ -1546,7 +1588,7 @@
     producer:
       name: HubSpot Inc.
       url: https://www.hubspot.com
-- 
+-
   user_agent: ICC-Crawler/2.0 (Mozilla-compatible; ; http://www.nict.go.jp/en/univ-com/plan/crawl.html)
   bot:
     name: ICC-Crawler
@@ -1555,7 +1597,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: IDG/IT (http://spaziodati.eu/)
   bot:
     name: IDG/IT
@@ -1564,7 +1606,7 @@
     producer:
       name: SpazioDati S.r.l.
       url: https://spaziodati.eu/
-- 
+-
   user_agent: iisbot/1.0 (+http://www.iis.net/iisbot.html)
   bot:
     name: IIS Site Analysis
@@ -1573,7 +1615,7 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
-- 
+-
   user_agent: IP-Guide.com Crawler/1.0 (https://ip-guide.com)
   bot:
     name: IP-Guide Crawler
@@ -1582,15 +1624,15 @@
     producer:
       name: ""
       url: https://ip-guide.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; ips-agent)
   bot:
     name: IPS Agent
-    category: crawler
+    category: Crawler
     producer:
       name: VeriSign, Inc
       url: http://www.verisign.com/
-- 
+-
   user_agent: Kaspersky Lab CFR link resolver cfradmins@kaspersky.com
   bot:
     name: Kaspersky
@@ -1599,7 +1641,7 @@
     producer:
       name: AO Kaspersky Lab
       url: https://www.kaspersky.com/
-- 
+-
   user_agent: kouio.com RSS reader
   bot:
     name: Kouio
@@ -1608,7 +1650,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: 'kouio.com RSS reader - 6 subscribers'
   bot:
     name: Kouio
@@ -1617,7 +1659,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)
   bot:
     name: LCC
@@ -1626,7 +1668,7 @@
     producer:
       name: Universität Leipzig
       url: https://www.uni-leipzig.de/
-- 
+-
   user_agent: 'ltx71 - (http://ltx71.com/)'
   bot:
     name: LTX71
@@ -1634,7 +1676,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: larbin_2.6.3 larbin2.6.3@unspecified.mail
   bot:
     name: Larbin web crawler
@@ -1643,7 +1685,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)"
   bot:
     name: "Let's Encrypt Validation"
@@ -1652,7 +1694,7 @@
     producer:
       name: "Let's Encrypt"
       url: https://letsencrypt.org
-- 
+-
   user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/61.0.3116.0 Mobile Safari/537.36 Chrome-Lighthouse
   bot:
     name: Lighthouse
@@ -1661,7 +1703,7 @@
     producer:
       name: Lighthouse
       url: https://developers.google.com/web/tools/lighthouse
-- 
+-
   user_agent: linkdex.com/v2.0 and linkdex.com/v2.1
   bot:
     name: Linkdex Bot
@@ -1670,7 +1712,7 @@
     producer:
       name: Mojeek Ltd.
       url: http://www.mojeek.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)
   bot:
     name: Linkdex Bot
@@ -1679,7 +1721,7 @@
     producer:
       name: Mojeek Ltd.
       url: http://www.mojeek.com
-- 
+-
   user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; linkdexbot-mobile/2.1; +http://www.linkdex.com/about/bots/)
   bot:
     name: Linkdex Bot
@@ -1688,7 +1730,7 @@
     producer:
       name: Mojeek Ltd.
       url: http://www.mojeek.com
-- 
+-
   user_agent: LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)
   bot:
     name: LinkedIn Bot
@@ -1697,7 +1739,7 @@
     producer:
       name: LinkedIn
       url: http://www.linkedin.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)
   bot:
     name: MJ12 Bot
@@ -1706,7 +1748,7 @@
     producer:
       name: Majestic-12
       url: http://majestic12.co.uk
-- 
+-
   user_agent: magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)
   bot:
     name: Magpie-Crawler
@@ -1715,7 +1757,7 @@
     producer:
       name: Brandwatch
       url: http://www.brandwatch.com
-- 
+-
   user_agent: MagpieRSS/0.72 (+http://magpierss.sf.net)
   bot:
     name: MagpieRSS
@@ -1724,7 +1766,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
   bot:
     name: Mail.Ru Bot
@@ -1733,7 +1775,7 @@
     producer:
       name: Mail.Ru Group
       url: http://corp.mail.ru
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +http://go.mail.ru/help/robots)
   bot:
     name: Mail.Ru Bot
@@ -1742,17 +1784,17 @@
     producer:
       name: Mail.Ru Group
       url: http://corp.mail.ru
-- 
+-
   user_agent: http.rb/2.2.2 (Mastodon/1.6.1; +https://mathtod.online/)
   bot:
     name: Mastodon Bot
     category: Social Media Agent
-- 
+-
   user_agent: http.rb/3.2.0 (Mastodon/2.4.3; +https://uwu.social/)
   bot:
     name: Mastodon Bot
     category: Social Media Agent
-- 
+-
   user_agent: Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)
   bot:
     name: Meanpath Bot
@@ -1761,13 +1803,13 @@
     producer:
       name: Meanpath
       url: http://www.meanpath.com
-- 
+-
   user_agent: MetaInspector/5.4.0 (+https://github.com/jaimeiniesta/metainspector)
   bot:
     name: MetaInspector
     category: Crawler
     url: https://github.com/jaimeiniesta/metainspector
-- 
+-
   user_agent: Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)
   bot:
     name: MetaJobBot
@@ -1776,7 +1818,7 @@
     producer:
       name: MetaJob
       url: http://www.metajob.at
-- 
+-
   user_agent: Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)
   bot:
     name: Mixrank Bot
@@ -1785,7 +1827,7 @@
     producer:
       name: Online Media Group, Inc.
       url: ""
-- 
+-
   user_agent: Mnogosearch-3.1.21
   bot:
     name: Mnogosearch
@@ -1794,7 +1836,7 @@
     producer:
       name: Lavtech.Com Corp.
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)
   bot:
     name: MojeekBot
@@ -1803,7 +1845,7 @@
     producer:
       name: Mojeek Ltd.
       url: http://www.mojeek.com
-- 
+-
   user_agent: 'Mozilla/5.0 (compatible; www.monitor.us - free monitoring service; http://www.monitor.us)'
   bot:
     name: Monitor.Us
@@ -1812,7 +1854,7 @@
     producer:
       name: Monitor.Us
       url: http://www.monitor.us
-- 
+-
   user_agent: munin/2.0.30-1 (libwww-perl/6.15)
   bot:
     name: Munin
@@ -1821,7 +1863,7 @@
     producer:
       name: Munin
       url: http://munin-monitoring.org/
-- 
+-
   user_agent: munin/http_loadtime
   bot:
     name: Munin
@@ -1830,7 +1872,7 @@
     producer:
       name: Munin
       url: http://munin-monitoring.org/
-- 
+-
   user_agent: nlcrawler/1.0 (+http://northernlight.com/)
   bot:
     name: NLCrawler
@@ -1839,7 +1881,7 @@
     producer:
       name: Northern Light
       url: http://northernlight.com
-- 
+-
   user_agent: check_http/v1.5 (nagios-plugins 1.5)
   bot:
     name: Nagios check_http
@@ -1848,7 +1890,7 @@
     producer:
       name: Nagios Plugins Development Team
       url: https://nagios.org
-- 
+-
   user_agent: NalezenCzBot/1.0 (http://www.nalezen.cz/about-crawler)
   bot:
     name: NalezenCzBot
@@ -1857,11 +1899,11 @@
     producer:
       name: Jaroslav Kuboš
       url: ""
-- 
+-
   user_agent: NetLyzer FastProbe
   bot:
     name: NetLyzer FastProbe
-- 
+-
   user_agent: Mozilla/4.0 (compatible; Netcraft Web Server Survey)
   bot:
     name: Netcraft Survey Bot
@@ -1870,7 +1912,7 @@
     producer:
       name: Netcraft
       url: http://www.netcraft.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)
   bot:
     name: Netcraft Survey Bot
@@ -1879,7 +1921,7 @@
     producer:
       name: Netcraft
       url: http://www.netcraft.com
-- 
+-
   user_agent: 'Netcraft SSL Server Survey - contact info@netcraft.com'
   bot:
     name: Netcraft Survey Bot
@@ -1888,7 +1930,7 @@
     producer:
       name: Netcraft
       url: http://www.netcraft.com
-- 
+-
   user_agent: Netvibes (http://www.netvibes.com)
   bot:
     name: Netvibes
@@ -1897,7 +1939,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: 'Netvibes (http://www.netvibes.com/; 8 subscribers; feedID: 2244192)'
   bot:
     name: Netvibes
@@ -1906,7 +1948,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: 'NewsBlur Favicon Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
   bot:
     name: NewsBlur
@@ -1915,7 +1957,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: 'NewsBlur Feed Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
   bot:
     name: NewsBlur
@@ -1924,7 +1966,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: NewsBlur Feed Finder (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
   bot:
     name: NewsBlur
@@ -1933,7 +1975,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: 'NewsBlur Page Fetcher - 7 subscribers - http://www.newsblur.com/site/3966817/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
   bot:
     name: NewsBlur
@@ -1942,7 +1984,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: NewsGatorOnline/2.0 (http://www.newsgator.com; 2 subscribers)
   bot:
     name: NewsGator
@@ -1951,7 +1993,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)
   bot:
     name: Nmap
@@ -1960,7 +2002,7 @@
     producer:
       name: Nmap
       url: https://nmap.org/
-- 
+-
   user_agent: 5acd734a-1ed3-4a17-ad9d-0b09ae7bc2d2/Nutch-1.7
   bot:
     name: Nutch-based Bot
@@ -1968,8 +2010,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: CB/Nutch-1.7
   bot:
     name: Nutch-based Bot
@@ -1977,8 +2019,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: Mozilla/5.0 (compatible; summers;)/Nutch-1.7
   bot:
     name: Nutch-based Bot
@@ -1986,8 +2028,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8/Nutch-1.7
   bot:
     name: Nutch-based Bot
@@ -1995,8 +2037,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: My Nutch Spider/Nutch-1.6
   bot:
     name: Nutch-based Bot
@@ -2004,8 +2046,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: MySpider/Nutch-2.2
   bot:
     name: Nutch-based Bot
@@ -2013,8 +2055,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: Nutch12/Nutch-1.2
   bot:
     name: Nutch-based Bot
@@ -2022,8 +2064,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: NutchSpider/Nutch-1.4
   bot:
     name: Nutch-based Bot
@@ -2031,8 +2073,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: spider/Nutch-1.5.1 (spider; http://www.xxx.com)
   bot:
     name: Nutch-based Bot
@@ -2040,8 +2082,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: Spider/Nutch-2.3-SNAPSHOT (Webcrawler)
   bot:
     name: Nutch-based Bot
@@ -2049,8 +2091,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: testnutch/Nutch-1.8
   bot:
     name: Nutch-based Bot
@@ -2058,8 +2100,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: Your Nutch Spider/Nutch-2.2.1
   bot:
     name: Nutch-based Bot
@@ -2067,8 +2109,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: your sipder name/Nutch-1.7
   bot:
     name: Nutch-based Bot
@@ -2076,8 +2118,8 @@
     url: https://nutch.apache.org
     producer:
       name: The Apache Software Foundation
-      url: http://www.apache.org/foundation/
-- 
+      url: https://www.apache.org/foundation/
+-
   user_agent: Nuzzel
   bot:
     name: Nuzzel
@@ -2085,11 +2127,11 @@
     producer:
       name: Nuzzel
       url: https://www.nuzzel.com/
-- 
+-
   user_agent: Octopus 1.0.2
   bot:
     name: Octopus
-- 
+-
   user_agent: omgili/0.5 +http://omgili.com
   bot:
     name: Omgili bot
@@ -2098,7 +2140,7 @@
     producer:
       name: Omgili
       url: http://www.omgili.com
-- 
+-
   user_agent: omgilibot/0.3 +http://www.omgili.com/Crawler.html
   bot:
     name: Omgili bot
@@ -2107,7 +2149,7 @@
     producer:
       name: Omgili
       url: http://www.omgili.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )
   bot:
     name: OpenLinkProfiler
@@ -2116,7 +2158,7 @@
     producer:
       name: Axandra GmbH
       url: http://www.axandra.com
-- 
+-
   user_agent: OpenWebSpider v0.1.4 (http://www.openwebspider.org/)
   bot:
     name: OpenWebSpider
@@ -2125,7 +2167,7 @@
     producer:
       name: OpenWebSpider Lab
       url: http://lab.openwebspider.org
-- 
+-
   user_agent: Mozilla/5.0 (compatible; OpenindexSpider; +http://www.openindex.io/en/webmasters/spider.html)
   bot:
     name: Openindex Spider
@@ -2134,7 +2176,7 @@
     producer:
       name: Openindex B.V.
       url: http://www.openindex.io
-- 
+-
   user_agent: Mozilla/5.0 (compatible; OrangeBot-Collector/2.0; support.orangebot@orange.com)
   bot:
     name: Orange Bot
@@ -2143,7 +2185,7 @@
     producer:
       name: Orange
       url: http://www.orange.fr
-- 
+-
   user_agent: Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com)
   bot:
     name: Orange Bot
@@ -2152,7 +2194,7 @@
     producer:
       name: Orange
       url: http://www.orange.fr
-- 
+-
   user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)
   bot:
     name: Orange Bot
@@ -2161,7 +2203,7 @@
     producer:
       name: Orange
       url: http://www.orange.fr
-- 
+-
   user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1;fr;rv:1.8.1) VoilaBotCollector BETA 0.1 (http://www.voila.com/)
   bot:
     name: Orange Bot
@@ -2170,7 +2212,7 @@
     producer:
       name: Orange
       url: http://www.orange.fr
-- 
+-
   user_agent: Mozilla/5.0 (Java) outbrain
   bot:
     name: Outbrain
@@ -2179,7 +2221,7 @@
     producer:
       name: Outbrain
       url: http://www.outbrain.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; phpservermon/3.1.1; +http://www.phpservermonitor.org)
   bot:
     name: PHP Server Monitor
@@ -2188,7 +2230,16 @@
     producer:
       name: PHP Server Monitor
       url: http://www.phpservermonitor.org/
-- 
+-
+  user_agent: Mozilla/5.0 (compatible; PRTG Network Monitor (www.paessler.com); Windows)
+  bot:
+    name: PRTG Network Monitor
+    category: Network Monitor
+    url: https://www.paessler.com/prtg
+    producer:
+      name: Paessler AG
+      url: https://www.paessler.com
+-
   user_agent: Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)
   bot:
     name: PaperLiBot
@@ -2197,13 +2248,19 @@
     producer:
       name: Smallrivers SA
       url: http://www.paper.li
-- 
+-
+  user_agent: Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://aspiegel.com/petalbot)
+  bot:
+    name: Petal Bot
+    category: Crawler
+    url: https://aspiegel.com/petalbot
+-
   user_agent: phantomas/1.11.0 (PhantomJS/1.9.8; linux x64)
   bot:
     name: Phantomas
     category: Site Monitor
     url: https://github.com/macbre/phantomas
-- 
+-
   user_agent: psbot-page (+http://www.picsearch.com/bot.html)
   bot:
     name: Picsearch bot
@@ -2212,7 +2269,7 @@
     producer:
       name: Picsearch
       url: http://www.picsearch.com
-- 
+-
   user_agent: psbot/0.1 (+http://www.picsearch.com/bot.html)
   bot:
     name: Picsearch bot
@@ -2221,7 +2278,7 @@
     producer:
       name: Picsearch
       url: http://www.picsearch.com
-- 
+-
   user_agent: Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
   bot:
     name: Pingdom Bot
@@ -2230,25 +2287,34 @@
     producer:
       name: Pingdom AB
       url: https://www.pingdom.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Pinterestbot/1.0; http://www.pinterest.com/bot.html)
   bot:
     name: Pinterest
-    url: http://www.pinterest.com/bot.html
+    url: https://help.pinterest.com/en/business/article/pinterest-crawler
     category: Crawler
     producer:
       name: Pinterest
-      url: http://www.pinterest.com/
-- 
+      url: https://www.pinterest.com/
+-
   user_agent: Pinterest/0.2 (+http://www.pinterest.com/)
   bot:
     name: Pinterest
-    url: http://www.pinterest.com/bot.html
+    url: https://help.pinterest.com/en/business/article/pinterest-crawler
+    category: Crawler
+    producer:
+      name: Pinterest
+      url: https://www.pinterest.com/
+-
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Pinterestbot/1.0;  https://www.pinterest.com/bot.html)
+  bot:
+    name: Pinterest
+    url: https://help.pinterest.com/en/business/article/pinterest-crawler
     category: Crawler
     producer:
       name: Pinterest
-      url: http://www.pinterest.com/
-- 
+      url: https://www.pinterest.com/
+-
   user_agent: PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)
   bot:
     name: PocketParser
@@ -2257,7 +2323,7 @@
     producer:
       name: Pocket
       url: https://getpocket.com/
-- 
+-
   user_agent: PritTorrent/1.0
   bot:
     name: PritTorrent
@@ -2266,7 +2332,7 @@
     producer:
       name: Bitlove
       url: http://bitlove.org/
-- 
+-
   user_agent: QuerySeekerSpider ( http://queryseeker.com/bot.html )
   bot:
     name: QuerySeekerSpider
@@ -2275,7 +2341,7 @@
     producer:
       name: QueryEye Inc.
       url: http://queryeye.com
-- 
+-
   user_agent: Quora Link Preview/1.0 (http://www.quora.com)
   bot:
     name: Quora Link Preview
@@ -2284,7 +2350,7 @@
     producer:
       name: Quora
       url: http://www.quora.com
-- 
+-
   user_agent: 'Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*'
   bot:
     name: Qwantify
@@ -2293,7 +2359,7 @@
     producer:
       name: Qwant Corporation
       url: https://www.qwant.com/
-- 
+-
   user_agent: ROI Hunter; https://api-dev.roihunter.com
   bot:
     name: ROI Hunter
@@ -2302,17 +2368,17 @@
     producer:
       name: Roihunter a.s.
       url: http://roihunter.com/
-- 
+-
   user_agent: RSSRadio (Push Notification Scanner;support@dorada.co.uk)
   bot:
     name: RSSRadio Bot
-- 
+-
   user_agent: Rainmeter WebParser plugin
   bot:
     name: Rainmeter
     category: Crawler
     url: https://www.rainmeter.net
-- 
+-
   user_agent: RamblerMail/6.0 (incompatible; ImageProxy/6.0)
   bot:
     name: RamblerMail Image Proxy
@@ -2321,7 +2387,7 @@
     producer:
       name: 'Rambler&Co'
       url: https://rambler-co.ru/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)
   bot:
     name: Reddit Bot
@@ -2330,7 +2396,12 @@
     producer:
       name: reddit inc.
       url: http://www.reddit.com
-- 
+-
+  user_agent: Robozilla/1.0
+  bot:
+    name: Robozilla
+    category: Crawler
+-
   user_agent: Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)
   bot:
     name: Rogerbot
@@ -2339,7 +2410,7 @@
     producer:
       name: SEOmoz, Inc.
       url: http://moz.com/
-- 
+-
   user_agent: rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)
   bot:
     name: Rogerbot
@@ -2348,7 +2419,7 @@
     producer:
       name: SEOmoz, Inc.
       url: http://moz.com/
-- 
+-
   user_agent: rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)
   bot:
     name: Rogerbot
@@ -2357,7 +2428,7 @@
     producer:
       name: SEOmoz, Inc.
       url: http://moz.com/
-- 
+-
   user_agent: SEOENGWorldBot/1.0 (+http://www.seoengine.com/seoengbot.htm)
   bot:
     name: SEOENGBot
@@ -2366,7 +2437,7 @@
     producer:
       name: SEO Engine
       url: http://www.seoengine.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)
   bot:
     name: SEOkicks-Robot
@@ -2375,7 +2446,7 @@
     producer:
       name: SEOkicks
       url: https://www.seokicks.de/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)
   bot:
     name: SISTRIX Crawler
@@ -2402,7 +2473,7 @@
     producer:
       name: SISTRIX GmbH
       url: http://www.sistrix.de
-- 
+-
   user_agent: Mozilla/5.0 (compatible; SISTRIX Optimizer; Uptime; +https://www.sistrix.com/faq/uptime)
   bot:
     name: SISTRIX Optimizer
@@ -2411,7 +2482,7 @@
     producer:
       name: SISTRIX GmbH
       url: http://www.sistrix.de
-- 
+-
   user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
   bot:
     name: SMTBot
@@ -2420,7 +2491,7 @@
     producer:
       name: SimilarTech Ltd.
       url: https://www.similartech.com/
-- 
+-
   user_agent: SSL Labs (https://www.ssllabs.com/about/assessment.html)
   bot:
     name: SSL Labs
@@ -2429,7 +2500,7 @@
     producer:
       name: SSL Labs
       url: https://www.ssllabs.com/about/assessment.html
-- 
+-
   user_agent: SafeDNSBot (https://www.safedns.com/searchbot)
   bot:
     name: SafeDNSBot
@@ -2438,13 +2509,13 @@
     producer:
       name: SafeDNS, Inc.
       url: https://www.safedns.com/
-- 
+-
   user_agent: Scrapy/1.0.3.post6+g2d688cd (+http://scrapy.org)
   bot:
     name: Scrapy
     category: Crawler
     url: http://scrapy.org
-- 
+-
   user_agent: Screaming Frog SEO Spider/2.22
   bot:
     name: Screaming Frog SEO Spider
@@ -2453,7 +2524,7 @@
     producer:
       name: Screaming Frog Ltd
       url: http://www.screamingfrog.co.uk
-- 
+-
   user_agent: ScreenerBot Crawler Beta 2.0 (+http://www.ScreenerBot.com)
   bot:
     name: ScreenerBot
@@ -2462,7 +2533,13 @@
     producer:
       name: ""
       url: ""
-- 
+-
+  user_agent: Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)
+  bot:
+    name: Semantic Scholar Bot
+    category: Crawler
+    url: https://www.semanticscholar.org/crawler
+-
   user_agent: Mozilla/5.0 (compatible; SemrushBot/0.97; +http://www.semrush.com/bot.html)
   bot:
     name: Semrush Bot
@@ -2471,7 +2548,7 @@
     producer:
       name: SEMrush
       url: http://www.semrush.com
-- 
+-
   user_agent: SensikaBot/x.33 (+http://sensika.com)
   bot:
     name: Sensika Bot
@@ -2480,24 +2557,36 @@
     producer:
       name: Sensika
       url: http://sensika.com
-- 
+-
   user_agent: sentry/8.6.0 (https://getsentry.com)
   bot:
     name: Sentry Bot
     producer:
       name: Sentry
       url: https://sentry.io
-- 
+-
+  user_agent: Seobility
+  bot:
+    name: Seobility
+    category: Crawler
+    url: 'https://www.seobility.net/en/faq/?category=crawling#!aboutourbot'
+-
   user_agent: Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)
   bot:
     name: Seoscanners.net
     category: Crawler
     url: ""
-- 
+-
+  user_agent: SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)
+  bot:
+    name: Serendeputy Bot
+    category: Crawler
+    url: https://serendeputy.com/about/serendeputy-bot
+-
   user_agent: Server Density Service Monitoring v2
   bot:
     name: Server Density
-- 
+-
   user_agent: Mozilla/5.0 (compatible; SeznamBot/3.1-test1; +http://fulltext.sblog.cz/)
   bot:
     name: Seznam Bot
@@ -2506,7 +2595,7 @@
     producer:
       name: Seznam.cz, a.s.
       url: http://www.seznam.cz/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://fulltext.sblog.cz/)
   bot:
     name: Seznam Bot
@@ -2515,7 +2604,7 @@
     producer:
       name: Seznam.cz, a.s.
       url: http://www.seznam.cz/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)
   bot:
     name: Seznam Bot
@@ -2524,7 +2613,7 @@
     producer:
       name: Seznam.cz, a.s.
       url: http://www.seznam.cz/
-- 
+-
   user_agent: SeznamBot/3.0 (+http://fulltext.sblog.cz/)
   bot:
     name: Seznam Bot
@@ -2533,7 +2622,7 @@
     producer:
       name: Seznam.cz, a.s.
       url: http://www.seznam.cz/
-- 
+-
   user_agent: Mozilla/5.0 SeznamEmailProxy/2.0.174
   bot:
     name: Seznam Email Proxy
@@ -2542,7 +2631,7 @@
     producer:
       name: Seznam.cz, a.s.
       url: http://www.seznam.cz/
-- 
+-
   user_agent: Seznam-Zbozi-robot/3.0
   bot:
     name: Seznam Zbozi.cz
@@ -2551,7 +2640,7 @@
     producer:
       name: Seznam.cz, a.s.
       url: https://www.zbozi.cz/
-- 
+-
   user_agent: Mozilla/5.0 (ShopAlike; LadenZeile) FeedBot
   bot:
     name: ShopAlike
@@ -2560,7 +2649,7 @@
     producer:
       name: Visual Meta
       url: https://www.shopalike.cz/
-- 
+-
   user_agent: ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)
   bot:
     name: ShopWiki
@@ -2569,7 +2658,7 @@
     producer:
       name: ShopWiki Corp.
       url: http://www.shopwiki.com
-- 
+-
   user_agent: shopify-partner-homepage-scraper
   bot:
     name: Shopify Partner
@@ -2578,7 +2667,7 @@
     producer:
       name: Shopify
       url: https://www.shopify.com/
-- 
+-
   user_agent: SilverReader/1.0; http://silverreader.com
   bot:
     name: SilverReader
@@ -2587,7 +2676,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: SimplePie/1.2.1-dev (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20130514092120
   bot:
     name: SimplePie
@@ -2596,7 +2685,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20121030175911
   bot:
     name: SimplePie
@@ -2605,7 +2694,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Site24x7
   bot:
     name: Site24x7 Website Monitoring
@@ -2614,13 +2703,13 @@
     producer:
       name: Site24x7
       url: https://www.site24x7.com
-- 
+-
   user_agent: SiteSucker for macOS/2.10.5
   bot:
     name: SiteSucker
     category: Crawler
     url: http://ricks-apps.com/osx/sitesucker/
-- 
+-
   user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com
   bot:
     name: Siteimprove
@@ -2629,7 +2718,7 @@
     producer:
       name: Siteimprove GmbH
       url: https://siteimprove.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
   bot:
     name: Siteimprove
@@ -2638,7 +2727,7 @@
     producer:
       name: Siteimprove GmbH
       url: https://siteimprove.com/
-- 
+-
   user_agent: sixy.ch/1.0
   bot:
     name: Sixy.ch
@@ -2647,7 +2736,7 @@
     producer:
       name: Manuel Kasper
       url: https://neon1.net/
-- 
+-
   user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5
   bot:
     name: Skype URI Preview
@@ -2656,7 +2745,7 @@
     producer:
       name: Skype Communications S.à.r.l.
       url: https://www.skype.com
-- 
+-
   user_agent: Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)
   bot:
     name: Slackbot
@@ -2665,7 +2754,7 @@
     producer:
       name: Slack Technologies
       url: http://slack.com
-- 
+-
   user_agent: 'AppEngine-Google; (+http://code.google.com/appengine; appid: s~snapchat-proxy)'
   bot:
     name: Snapchat Proxy
@@ -2674,7 +2763,7 @@
     producer:
       name: Snapchat Inc.
       url: https://www.snapchat.com
-- 
+-
   user_agent: New-Sogou-Spider/1.0 (compatible; MSIE 5.5; Windows 98)
   bot:
     name: Sogou Spider
@@ -2683,7 +2772,7 @@
     producer:
       name: Sohu, Inc.
       url: http://www.sogou.com
-- 
+-
   user_agent: Sogou inst spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm
   bot:
     name: Sogou Spider
@@ -2692,7 +2781,7 @@
     producer:
       name: Sohu, Inc.
       url: http://www.sogou.com
-- 
+-
   user_agent: Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm
   bot:
     name: Sogou Spider
@@ -2701,7 +2790,7 @@
     producer:
       name: Sohu, Inc.
       url: http://www.sogou.com
-- 
+-
   user_agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm
   bot:
     name: Sogou Spider
@@ -2710,7 +2799,7 @@
     producer:
       name: Sohu, Inc.
       url: http://www.sogou.com
-- 
+-
   user_agent: Sosospider+(+http://help.soso.com/webspider.htm)
   bot:
     name: Soso Spider
@@ -2719,13 +2808,13 @@
     producer:
       name: Tencent Holdings
       url: http://www.soso.com
-- 
+-
   user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Sparkler/0.2.0-SNAPSHOT
   bot:
     name: Sparkler
     category: Crawler
     url: https://github.com/USCDataScience/sparkler
-- 
+-
   user_agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.19; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/2010040121 Firefox/3.0.19
   bot:
     name: Spinn3r
@@ -2734,32 +2823,26 @@
     producer:
       name: Tailrank Inc
       url: http://spinn3r.com
-- 
+-
   user_agent: Spotify/1.0
   bot:
     name: Spotify
     producer:
       name: Spotify
       url: https://www.spotify.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; SputnikBot/2.2)
   bot:
     name: Sputnik Bot
-    category: ""
+    category: "Crawler"
     url: ""
-    producer:
-      name: ""
-      url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; SputnikImageBot/2.2)
   bot:
-    name: Sputnik Bot
-    category: ""
+    name: Sputnik Image Bot
+    category: "Crawler"
     url: ""
-    producer:
-      name: ""
-      url: ""
-- 
+-
   user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) (compatible; Startpagina-Linkchecker/1.0; +https://www.startpagina.nl/linkchecker)
   bot:
     name: Startpagina Linkchecker
@@ -2768,7 +2851,7 @@
     producer:
       name: Startpagina B.V.
       url: https://www.startpagina.nl/
-- 
+-
   user_agent: Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake)
   bot:
     name: StatusCake
@@ -2777,7 +2860,7 @@
     producer:
       name: StatusCake
       url: https://www.statuscake.com
-- 
+-
   user_agent: 'Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch!'
   bot:
     name: Superfeedr Bot
@@ -2786,7 +2869,7 @@
     producer:
       name: Superfeedr
       url: https://superfeedr.com/
-- 
+-
   user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)
   bot:
     name: Survey Bot
@@ -2795,7 +2878,7 @@
     producer:
       name: Domain Tools
       url: http://www.domaintools.com
-- 
+-
   user_agent: TLSProbe/1.0 (+https://scan.trustnet.venafi.com/)
   bot:
     name: TLSProbe
@@ -2804,23 +2887,23 @@
     producer:
       name: Venafi TrustNet
       url: https://www.venafi.com
-- 
+-
   user_agent: Tarmot Gezgin/1.0 (compatible; TarmotGezgin/1.1; +http://www.tarmot.com/gezgin)
   bot:
     name: Tarmot Gezgin
     url: http://www.tarmot.com/gezgin/
     category: Search bot
-- 
+-
   user_agent: TelegramBot (like TwitterBot)
   bot:
     name: TelegramBot
     url: https://telegram.org/blog/bot-revolution
-- 
+-
   user_agent: The Knowledge AI
   bot:
     name: The Knowledge AI
     category: Crawler
-- 
+-
   user_agent: TinEye-bot/0.02 (see http://www.tineye.com/crawler.html)
   bot:
     name: TinEye Crawler
@@ -2829,7 +2912,7 @@
     producer:
       name: Idée Inc.
       url: http://ideeinc.com
-- 
+-
   user_agent: Tiny Tiny RSS/1.10 (http://tt-rss.org/)
   bot:
     name: Tiny Tiny RSS
@@ -2838,7 +2921,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Tiny Tiny RSS/1.11.4c63934 (http://tt-rss.org/)
   bot:
     name: Tiny Tiny RSS
@@ -2847,7 +2930,7 @@
     producer:
       name: ""
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; tracemyfile/1.0)
   bot:
     name: TraceMyFile
@@ -2856,7 +2939,7 @@
     producer:
       name: Idee Inc.
       url: http://ideeinc.com/
-- 
+-
   user_agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11
   bot:
     name: Trendiction Bot
@@ -2865,7 +2948,7 @@
     producer:
       name: Talkwalker Inc.
       url: http://www.talkwalker.com
-- 
+-
   user_agent: TurnitinBot/3.0 (http://www.turnitin.com/robot/crawlerinfo.html)
   bot:
     name: TurnitinBot
@@ -2874,7 +2957,7 @@
     producer:
       name: iParadigms, LLC.
       url: http://www.turnitin.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; TweetedTimes Bot/1.0; +http://tweetedtimes.com)
   bot:
     name: TweetedTimes Bot
@@ -2883,7 +2966,7 @@
     producer:
       name: TweetedTimes
       url: http://tweetedtimes.com/
-- 
+-
   user_agent: TweetedTimes Bot/1.0 (Mozilla/5.0 Compatible, +http://tweetedtimes.com)
   bot:
     name: TweetedTimes Bot
@@ -2892,7 +2975,7 @@
     producer:
       name: TweetedTimes
       url: http://tweetedtimes.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; TweetmemeBot/3.0; +http://tweetmeme.com/)
   bot:
     name: Tweetmeme Bot
@@ -2901,7 +2984,7 @@
     producer:
       name: Mediasift
       url: ""
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Twingly Recon; twingly.com)
   bot:
     name: Twingly Recon
@@ -2909,7 +2992,7 @@
     producer:
       name: Twingly
       url: https://www.twingly.com
-- 
+-
   user_agent: Twitterbot/1.0
   bot:
     name: Twitterbot
@@ -2918,7 +3001,7 @@
     producer:
       name: Twitter
       url: http://www.twitter.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)
   bot:
     name: URLAppendBot
@@ -2927,7 +3010,7 @@
     producer:
       name: Profound Networks
       url: http://www.profound.net
-- 
+-
   user_agent: 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 (via secureurl.fwdcdn.com - mail.ukr.net proxy)'
   bot:
     name: UkrNet Mail Proxy
@@ -2936,7 +3019,7 @@
     producer:
       name: UkrNet Ltd
       url: https://www.ukr.net/
-- 
+-
   user_agent: UniversalFeedParser/5.2.1 +https://code.google.com/p/feedparser/
   bot:
     name: UniversalFeedParser
@@ -2945,7 +3028,7 @@
     producer:
       name: Kurt McKee
       url: https://github.com/kurtmckee
-- 
+-
   user_agent: Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)
   bot:
     name: Uptime Robot
@@ -2954,7 +3037,7 @@
     producer:
       name: Uptime Robot
       url: http://uptimerobot.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)
   bot:
     name: Uptimebot
@@ -2963,7 +3046,7 @@
     producer:
       name: Uptime
       url: https://uptime.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)
   bot:
     name: VK Share Button
@@ -2972,7 +3055,7 @@
     producer:
       name: VK
       url: http://vk.com/
-- 
+-
   user_agent: 'Mozilla/4.0 (compatible;  Vagabondo/4.0; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)'
   bot:
     name: Vagabondo
@@ -2981,7 +3064,19 @@
     producer:
       name: WiseGuys
       url: http://www.wise-guys.nl/
-- 
+-
+  user_agent: Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)
+  bot:
+    name: Velen Public Web Crawler
+    category: Crawler
+    url: https://hunter.io/robot
+-
+  user_agent: Vercelbot (+https://vercel.com)
+  bot:
+    name: Vercel Bot
+    category: Service bot
+    url: https://vercel.com
+-
   user_agent: Mozilla/5.0 (compatible; VSMCrawler; http://www.visualsitemapper.com/crawler/)
   bot:
     name: Visual Site Mapper Crawler
@@ -2990,7 +3085,7 @@
     producer:
       name: Alentum Software Ltd.
       url: http://www.alentum.com
-- 
+-
   user_agent: 'Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0 (See <http://validator.w3.org/services>)'
   bot:
     name: W3C CSS Validator
@@ -2999,7 +3094,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: W3C_I18n-Checker/1.0 (http://validator.w3.org/services)
   bot:
     name: W3C I18N Checker
@@ -3008,7 +3103,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: 'W3C-checklink/4.0 [4.4] libwww-perl/5.803'
   bot:
     name: W3C Link Checker
@@ -3017,7 +3112,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: W3C-checklink/4.81 libwww-perl/5.836
   bot:
     name: W3C Link Checker
@@ -3026,7 +3121,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: Validator.nu/LV http://validator.w3.org/services
   bot:
     name: W3C Markup Validation Service
@@ -3035,7 +3130,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: W3C_Validator/1.3 http://validator.w3.org/services
   bot:
     name: W3C Markup Validation Service
@@ -3044,7 +3139,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: W3C_Validator/1.767
   bot:
     name: W3C Markup Validation Service
@@ -3053,7 +3148,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: W3C-mobileOK/DDC-1.0 (see http://www.w3.org/2006/07/mobileok-ddc)
   bot:
     name: W3C MobileOK Checker
@@ -3062,7 +3157,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: W3C_Unicorn/1.0 (http://validator.w3.org/services)
   bot:
     name: W3C Unified Validator
@@ -3071,7 +3166,7 @@
     producer:
       name: W3C
       url: http://www.w3.org
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Wappalyzer; +https://github.com/AliasIO/Wappalyzer)
   bot:
     name: Wappalyzer
@@ -3079,7 +3174,7 @@
     producer:
       name: AliasIO
       url: https://github.com/AliasIO
-- 
+-
   user_agent: WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)
   bot:
     name: WeSEE:Search
@@ -3088,13 +3183,13 @@
     producer:
       name: WeSEE Ltd
       url: http://www.wesee.com
-- 
+-
   user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4) Build/MPJ24.139-64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Mobile Safari/537.36 PTST/180829.190838
   bot:
     name: WebPageTest
     category: Site Monitor
     url: https://www.webpagetest.org
-- 
+-
   user_agent: websitepulse checker/1.1 (compatible; MSIE 5.5; Netscape 4.75; Linux)
   bot:
     name: WebSitePulse
@@ -3103,7 +3198,7 @@
     producer:
       name: WebSitePulse
       url: http://www.websitepulse.com/
-- 
+-
   user_agent: WebbCrawler 1.0 ( http://badcheese.com/crawler.html )
   bot:
     name: WebbCrawler
@@ -3112,7 +3207,7 @@
     producer:
       name: Steve Webb
       url: http://badcheese.com
-- 
+-
   user_agent: weborama-fetcher (+http://www.weborama.com)
   bot:
     name: Weborama
@@ -3121,7 +3216,7 @@
     producer:
       name: Weborama SA
       url: https://weborama.com/
-- 
+-
   user_agent: WikiDo/1.1 (http://wikido.com; crawler@wikido.com)
   bot:
     name: WikiDo
@@ -3130,7 +3225,7 @@
     producer:
       name: Fotolitografie Fiorentine di Becchi Antonio s.n.c.
       url: https://www.wikido.com/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)
   bot:
     name: WooRank
@@ -3139,7 +3234,7 @@
     producer:
       name: WooRank sprl
       url: https://www.woorank.com/
-- 
+-
   user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)
   bot:
     name: WooRank
@@ -3148,7 +3243,7 @@
     producer:
       name: WooRank sprl
       url: https://www.woorank.com/
-- 
+-
   user_agent: WordPress/4.7.2; https://example.com
   bot:
     name: WordPress
@@ -3157,7 +3252,7 @@
     producer:
       name: Wordpress.org
       url: https://wordpress.org/
-- 
+-
   user_agent: Wotbox/2.01 (+http://www.wotbox.com/bot/)
   bot:
     name: Wotbox
@@ -3166,7 +3261,7 @@
     producer:
       name: Wotbox
       url: http://www.wotbox.com
-- 
+-
   user_agent: XenForo/2.x (https://www.example.com)
   bot:
     name: XenForo
@@ -3175,7 +3270,7 @@
     producer:
       name: XenForo Ltd.
       url: https://xenforo.com/
-- 
+-
   user_agent: yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
   bot:
     name: YaCy
@@ -3184,7 +3279,7 @@
     producer:
       name: YaCy
       url: http://yacy.net
-- 
+-
   user_agent: yacybot (freeworld/global; x86 Windows XP 5.1; java 1.7.0_21; GMT+04:00/ru) http://yacy.net/bot.html
   bot:
     name: YaCy
@@ -3193,7 +3288,7 @@
     producer:
       name: YaCy
       url: http://yacy.net
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)
   bot:
     name: Yahoo Gemini
@@ -3202,7 +3297,7 @@
     producer:
       name: Yahoo! Inc.
       url: http://www.yahoo.com
-- 
+-
   user_agent: Y!J-BRW/1.0 (https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716)
   bot:
     name: Yahoo! Japan BRW
@@ -3211,7 +3306,7 @@
     producer:
       name: Yahoo! Japan Corp.
       url: https://www.yahoo.co.jp/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)
   bot:
     name: Yahoo! Link Preview
@@ -3220,7 +3315,7 @@
     producer:
       name: Yahoo! Inc.
       url: http://www.yahoo.com
-- 
+-
   user_agent: Yahoo:LinkExpander:Slingstone
   bot:
     name: Yahoo! Link Preview
@@ -3229,7 +3324,7 @@
     producer:
       name: Yahoo! Inc.
       url: http://www.yahoo.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp) NOT Firefox/3.5
   bot:
     name: Yahoo! Slurp
@@ -3238,7 +3333,7 @@
     producer:
       name: Yahoo! Inc.
       url: http://www.yahoo.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
   bot:
     name: Yahoo! Slurp
@@ -3247,7 +3342,7 @@
     producer:
       name: Yahoo! Inc.
       url: http://www.yahoo.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; Linux i686; Yandex.Gazeta Bot/1.0; +http://gazeta.yandex.ru)
   bot:
     name: Yandex Bot
@@ -3256,7 +3351,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YaDirectFetcher/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3265,7 +3360,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3274,7 +3369,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3283,7 +3378,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; B; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3292,7 +3387,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; B; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3301,7 +3396,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3310,7 +3405,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3319,7 +3414,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3328,7 +3423,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3337,7 +3432,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3346,7 +3441,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3355,7 +3450,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3364,7 +3459,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3373,7 +3468,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexNewslinks; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3382,7 +3477,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3391,7 +3486,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3400,7 +3495,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3409,7 +3504,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3418,7 +3513,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3427,7 +3522,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3436,7 +3531,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3445,7 +3540,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YandexVerticals/1.0; http://yandex.com/bots)
   bot:
     name: Yandex Bot
@@ -3454,7 +3549,7 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; NaverJapan/1.0; +http://corp.naver.jp/)
   bot:
     name: Yeti/Naverbot
@@ -3463,7 +3558,7 @@
     producer:
       name: Naver
       url: http://www.naver.com
-- 
+-
   user_agent: Yeti/1.1 (Naver Corp.; http://help.naver.com/robots/)
   bot:
     name: Yeti/Naverbot
@@ -3472,7 +3567,7 @@
     producer:
       name: Naver
       url: http://www.naver.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )
   bot:
     name: Youdao Bot
@@ -3481,13 +3576,13 @@
     producer:
       name: NetEase, Inc.
       url: http://corp.163.com
-- 
+-
   user_agent: YOURLS v1.5.1 +http://yourls.org/ (running on http://fhort.com)
   bot:
     name: Yourls
     category: Crawler
     url: http://yourls.org
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YRSpider; +http://www.yunrang.com/yrspider.html)
   bot:
     name: Yunyun Bot
@@ -3496,7 +3591,7 @@
     producer:
       name: YunYun
       url: http://www.yunyun.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YRSpider; +http://www.yunyun.com/SiteInfo.php?r=about)
   bot:
     name: Yunyun Bot
@@ -3505,7 +3600,7 @@
     producer:
       name: YunYun
       url: http://www.yunyun.com
-- 
+-
   user_agent: Mozilla/5.0 (compatible; YYSpider; +http://www.yunyun.com/spider.html)
   bot:
     name: Yunyun Bot
@@ -3514,21 +3609,21 @@
     producer:
       name: YunYun
       url: http://www.yunyun.com
-- 
+-
   user_agent: Zao/0.1 (http://www.kototol.org/zao)
   bot:
     name: Zao
     category: Crawler
-- 
+-
   user_agent: zelist.ro feed parser (+http://www.zelist.ro)
   bot:
     name: Ze List
-    category: Feed Fetcher
     url: https://www.zelist.ro/
+    category: Feed Fetcher
     producer:
       name: Treeworks SRL
       url: https://www.tree.ro/
-- 
+-
   user_agent: Zookabot/2.5;++http://zookabot.com
   bot:
     name: Zookabot
@@ -3537,7 +3632,7 @@
     producer:
       name: Hwacha ApS
       url: http://hwacha.dk
-- 
+-
   user_agent: Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)
   bot:
     name: ZumBot
@@ -3546,7 +3641,7 @@
     producer:
       name: ZUM internet
       url: http://www.zuminternet.com/
-- 
+-
   user_agent: ZumBot/1.0 (ZUM Search; http://help.zum.com/inquiry)
   bot:
     name: ZumBot
@@ -3555,106 +3650,106 @@
     producer:
       name: ZUM internet
       url: http://www.zuminternet.com/
-- 
+-
   user_agent: AhrefsBot.Feeds v0.1; http://ahrefs.com/
   bot:
     name: aHrefs Bot
     category: Crawler
-    url: http://ahrefs.com/robot
+    url: https://ahrefs.com/robot
     producer:
       name: Ahrefs Pte Ltd
-      url: http://ahrefs.com/robot
-- 
+      url: https://ahrefs.com/robot
+-
   user_agent: Mozilla/5.0 (compatible; AhrefsBot/3.1; +http://ahrefs.com/robot/
   bot:
     name: aHrefs Bot
     category: Crawler
-    url: http://ahrefs.com/robot
+    url: https://ahrefs.com/robot
     producer:
       name: Ahrefs Pte Ltd
-      url: http://ahrefs.com/robot
-- 
+      url: https://ahrefs.com/robot
+-
   user_agent: Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)
   bot:
     name: aHrefs Bot
     category: Crawler
-    url: http://ahrefs.com/robot
+    url: https://ahrefs.com/robot
     producer:
       name: Ahrefs Pte Ltd
-      url: http://ahrefs.com/robot
-- 
+      url: https://ahrefs.com/robot
+-
   user_agent: 'Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/) AppEngine-Google; (+http://code.google.com/appengine; appid: s~proxyfile1-hrd)'
   bot:
     name: aHrefs Bot
     category: Crawler
-    url: http://ahrefs.com/robot
+    url: https://ahrefs.com/robot
     producer:
       name: Ahrefs Pte Ltd
-      url: http://ahrefs.com/robot
-- 
+      url: https://ahrefs.com/robot
+-
   user_agent: Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)
   bot:
     name: archive.org bot
     category: Crawler
-    url: http://www.archive.org/details/archive.org_bot
+    url: https://archive.org/details/archive.org_bot
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +http://pandora.nla.gov.au/crawl.html)
   bot:
     name: archive.org bot
     category: Crawler
-    url: http://www.archive.org/details/archive.org_bot
+    url: https://archive.org/details/archive.org_bot
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; archive.org_bot; Archive-It; +http://archive-it.org/files/site-owners.html)
   bot:
     name: archive.org bot
     category: Crawler
-    url: http://www.archive.org/details/archive.org_bot
+    url: https://archive.org/details/archive.org_bot
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)
   bot:
     name: archive.org bot
     category: Crawler
-    url: http://www.archive.org/details/archive.org_bot
+    url: https://archive.org/details/archive.org_bot
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20131029-0036 +http://archive.org/details/archive.org_bot)
   bot:
     name: archive.org bot
     category: Crawler
-    url: http://www.archive.org/details/archive.org_bot
+    url: https://archive.org/details/archive.org_bot
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)
   bot:
     name: archive.org bot
     category: Crawler
-    url: http://www.archive.org/details/archive.org_bot
+    url: https://archive.org/details/archive.org_bot
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: Mozilla/5.0 (compatible; special_archiver/3.2.0 +http://www.loc.gov/webarchiving/notice_to_webmasters.html)
   bot:
     name: archive.org bot
     category: Crawler
-    url: http://www.archive.org/details/archive.org_bot
+    url: https://archive.org/details/archive.org_bot
     producer:
       name: The Internet Archive
-      url: http://www.archive.org
-- 
+      url: https://archive.org
+-
   user_agent: CCBot/2.0 (http://commoncrawl.org/faq/)
   bot:
     name: ccBot crawler
@@ -3663,7 +3758,7 @@
     producer:
       name: reddit inc.
       url: http://www.reddit.com
-- 
+-
   user_agent: Mozilla/5.0 eCairn-Grabber/1.0 (+http://ecairn.com/grabber)
   bot:
     name: eCairn-Grabber
@@ -3671,7 +3766,7 @@
     producer:
       name: eCairn
       url: https://ecairn.com
-- 
+-
   user_agent: eZ Publish Link Validator
   bot:
     name: eZ Publish Link Validator
@@ -3680,7 +3775,7 @@
     producer:
       name: eZ Systems AS
       url: https://ez.no/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; evc-batch/2.0)
   bot:
     name: evc-batch
@@ -3689,13 +3784,13 @@
     producer:
       name: eVenture Capital Partners II, LLC
       url: http://www.eventures.vc/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; inoreader.com; 2 subscribers)
   bot:
     name: inoreader
     category: Feed Reader
     url: https://www.inoreader.com
-- 
+-
   user_agent: masscan/1.0 (https://github.com/robertdavidgraham/masscan)
   bot:
     name: masscan
@@ -3704,21 +3799,21 @@
     producer:
       name: Robert Graham
       url: https://github.com/robertdavidgraham
-- 
+-
   user_agent: 'Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com'
   bot:
     name: nbertaupete95
     category: Crawler
-- 
+-
   user_agent: Mozilla/5.0 (compatible; oBot/2.3.1; http://www.xforce-security.com/crawler/)
   bot:
     name: oBot
     category: Search bot
-    url: http://www.xforce-security.com/crawler/
+    url: https://www.xforce-security.com/crawler/
     producer:
       name: 'IBM Germany Research & Development GmbH'
       url: https://exchange.xforce.ibmcloud.com/
-- 
+-
   user_agent: 'sqlmap/1.1.8.2#dev (http://sqlmap.org)'
   bot:
     name: sqlmap
@@ -3727,87 +3822,613 @@
     producer:
       name: sqlmap
       url: http://sqlmap.org/
-- 
+-
   user_agent: Mozilla/5.0 (compatible; theoldreader.com; 1 subscribers; feed-id=aaa)
   bot:
     name: theoldreader
     category: Feed Reader
     url: https://theoldreader.com
-
 -
-  user_agent: Seobility
+  user_agent: Sprinklr 2.0
   bot:
-    name: Seobility
+    name: Sprinklr
     category: Crawler
-    url: 'https://www.seobility.net/en/faq/?category=crawling#!aboutourbot'
-
+    url: ""
+    producer:
+      name: Sprinklr, Inc.
+      url: https://www.sprinklr.com/
 -
-  user_agent: Vercelbot (+https://vercel.com)
+  user_agent: Mozilla/5.0 (X11; Linux x86_64)AppleWebKit/537.36 (KHTML, like Gecko; Google-Ads-Qualify)Chrome/85.0.4183.140 Safari/537.36
   bot:
-    name: Vercel Bot
-    category: Service bot
-    url: https://vercel.com
-
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 -
-  user_agent: Grammarly/1.0 (http://www.grammarly.com)
+  user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36
   bot:
-    name: Grammarly
-    category: Service bot
-    url: http://www.grammarly.com
-
+    name: BDCbot
+    category: Crawler
+    url: https://bigweb.bigdatacorp.com.br/pages/faq.aspx
+    producer:
+      name: BIG Data Solucoes Em Tecnologia de Informatica LTDA
+      url: https://bigdatacorp.com.br/
 -
-  user_agent: Robozilla/1.0
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36 CloudflareDiagnostics/1.0
   bot:
-    name: Robozilla
+    name: Cloudflare Diagnostics
+    category: Site Monitor
+    url: https://www.cloudflare.com/
+    producer:
+      name: Cloudflare
+      url: https://www.cloudflare.com
+-
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) adbeat.com/policy AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/69.0.3497.105 Mobile/15E148 Safari/605.1
+  bot:
+    name: Adbeat
     category: Crawler
-
+    url: https://www.adbeat.com/operation_policy
+    producer:
+      name: PPC Labs LLC
+      url: https://www.adbeat.com/
 -
-  user_agent: Mozilla/5.0 (compatible; Domains Project/1.1.0; +https://domainsproject.org)
+  user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BW/1.1; bit.ly/2W6Px8S) Chrome/84.0.4147.105 Safari/537.36
   bot:
-    name: Domains Project
+    name: BuiltWith
     category: Crawler
-    url: https://domainsproject.org
-
+    url: https://builtwith.com/biup
+    producer:
+      name: BuiltWith Pty Ltd
+      url: https://builtwith.com/
 -
-  user_agent: Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://aspiegel.com/petalbot)
+  user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 (+https://whatis.contentkingapp.com)
   bot:
-    name: Petal Bot
+    name: ContentKing
+    category: Site Monitor
+    url: https://whatis.contentkingapp.com/
+    producer:
+      name: ContentKing BV
+      url: https://www.contentkingapp.com/
+-
+  user_agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36 MicroAdBot/1.1 (https://www.microad.co.jp/contact/)
+  bot:
+    name: MicroAdBot
     category: Crawler
-    url: https://aspiegel.com/petalbot
-
+    url: https://www.microad.co.jp/
+    producer:
+      name: MicroAd, Inc.
+      url: https://www.microad.co.jp/
 -
-  user_agent: SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)
+  user_agent: Mozilla/5.0 (compatible; PingAdmin.Ru/1.2; +http://pingadmin.ru/free_test/)
   bot:
-    name: Serendeputy Bot
+    name: PingAdmin.Ru
+    category: Site Monitor
+    url: https://ping-admin.ru/
+-
+  user_agent: Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)
+  bot:
+    name: WebDataStats
     category: Crawler
-    url: http://serendeputy.com/about/serendeputy-bot
-
+    url: https://webdatastats.com/policy.html
+    producer:
+      name: WebTehRazrabotka LLC
+      url: https://webdatastats.com/
 -
-  user_agent: ias-va/3.1 (+https://www.admantx.com/service-fetcher.html)
+  user_agent: Mozilla/5.0 (compatible; parse.ly scraper/0.14; +http://parsely.com)
   bot:
-    name: ADmantX Service Fetcher
-    category: Service bot
-    url: https://www.admantx.com/service-fetcher.html
-
+    name: parse.ly
+    category: Crawler
+    url: https://www.parse.ly/help/integration/crawler
+    producer:
+      name: Parsely, Inc.
+      url: https://www.parse.ly/
 -
-  user_agent: Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)
+  user_agent: Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)
   bot:
-    name: Semantic Scholar Bot
+    name: Nimbostratus Bot
+    category: Site Monitor
+    url: http://cloudsystemnetworks.com
+-
+  user_agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.17) Gecko/20110515 HeartRails_Capture/1.0.4 (+http://capture.heartrails.com/) Namoroka/3.6.17
+  bot:
+    name: Heart Rails Capture
+    category: Service Agent
+    url: http://capture.heartrails.com
+-
+  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36 http://notifyninja.com/monitoring
+  bot:
+    name: Notify Ninja
+    category: Site Monitor
+    url: http://notifyninja.com
+-
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) Project-Resonance (http://project-resonance.com/) (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36
+  bot:
+    name: Project Resonance
     category: Crawler
-    url: https://www.semanticscholar.org/crawler
-
+    url: http://project-resonance.com
 -
-  user_agent: Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)
+  user_agent: Mozilla/5.0 (compatible; DataXu/1.0; +http://dataxu.com)
   bot:
-    name: Velen Public Web Crawler
+    name: DataXu
+    category: Service Agent
+    url: https://advertising.roku.com/dataxu
+    producer:
+      name: Roku, Inc.
+      url: https://roku.com
+-
+  user_agent: Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) PingdomTMS/0.8.5 Safari/534.34
+  bot:
+    name: Pingdom Bot
+    category: Site Monitor
+    url: ""
+    producer:
+      name: Pingdom AB
+      url: https://www.pingdom.com
+-
+  user_agent: Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)
+  bot:
+    name: Cocolyzebot
     category: Crawler
-    url: https://hunter.io/robot
-
+    url: https://cocolyze.com/en/cocolyzebot
+    producer:
+      name: VSI INNOVATION SAS
+      url: https://vsi-innovation.com/
 -
-  user_agent: Barkrowler/0.9 (+http://www.exensa.com/crawl)
+  user_agent: veryhip (http://veryhip.com/)
   bot:
-    name: Barkrowler
+    name: VeryHip
     category: Crawler
-    url: http://www.exensa.com/crawl
-
-
+    url: https://veryhip.com/
+    producer:
+      name: VeryHip
+      url: https://veryhip.com/
+-
+  user_agent: Mozilla/5.0 (compatible; LinkpadBot/2.3; +http://linkpad.org/robot/)
+  bot:
+    name: LinkpadBot
+    category: Crawler
+    url: https://www.linkpad.org/
+    producer:
+      name: Solomono LLC
+      url: https://www.linkpad.org/
+-
+  user_agent: Mozilla/4.0 (compatible; MuscatFerret/1.7; http://www.webtop.com/)
+  bot:
+    name: MuscatFerret
+    category: Crawler
+    url: http://www.webtop.com/
+-
+  user_agent: Quora-Bot/1.0 (http://www.quora.com)
+  bot:
+    name: Quora Bot
+    category: Crawler
+    url: ""
+    producer:
+      name: Quora
+      url: https://www.quora.com/
+-
+  user_agent: PageThing.com
+  bot:
+    name: PageThing
+    category: Crawler
+    url: https://www.pagething.com/
+    producer:
+      name: SPECIALNOISE LTD
+      url: https://www.specialnoise.com/
+-
+  user_agent: ArchiveBox/0.5.3 (+https://github.com/ArchiveBox/ArchiveBox/)
+  bot:
+    name: ArchiveBox
+    url: https://archivebox.io/
+    category: Crawler
+    producer:
+      name: ""
+      url: ""
+-
+  user_agent: Choosito/1.0 +http://www.choosito.com/
+  bot:
+    name: Choosito
+    url: https://www.choosito.com/
+    category: Crawler
+    producer:
+      name: Choosito! Inc.
+      url: https://www.choosito.com/
+-
+  user_agent: datagnionbot (+http://www.datagnion.com/bot.html)
+  bot:
+    name: datagnionbot
+    url: https://www.datagnion.com/bot.html
+    category: Crawler
+    producer:
+      name: DATAGNION GMBH
+      url: https://www.datagnion.com/
+-
+  user_agent: InterNaetBoten/0.99 (http://www.xn--internt-bxa.nu)
+  bot:
+    name: Generic Bot
+-
+  user_agent: EasyBib AutoCite (http://autocite-info.citation-api.com/)
+  bot:
+    name: Generic Bot
+-
+  user_agent: Bidtellect/0.0.643.0
+  bot:
+    name: Generic Bot
+-
+  user_agent: Mozilla/5.0 (compatible; meg/0.2; +https://github.com/tomnomnom/meg)
+  bot:
+    name: Generic Bot
+-
+  user_agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) WhatCMS/1.0
+  bot:
+    name: WhatCMS
+    url: https://whatcms.org/
+    category: Crawler
+    producer:
+      name: "Nineteen Ten LLC"
+      url: "https://whatcms.org/"
+-
+  user_agent: httpx - Open-source project (github.com/projectdiscovery/httpx)
+  bot:
+    name: httpx
+    url: https://github.com/projectdiscovery/httpx
+    category: Crawler
+    producer:
+      name: ""
+      url: ""
+-
+  user_agent: 'Expanse indexes the network perimeters of our customers. If you have any questions or concerns, please reach out to: scaninfo@expanseinc.com'
+  bot:
+    name: Expanse
+    category: Security Checker
+    url: https://expanse.co/
+    producer:
+      name: Expanse Inc.
+      url: https://expanse.co/
+-
+  user_agent: 'HuaweiWebCatBot/6.0) (To acquire the allowed html pages as reliable information of URL categorization in the automatic process for Huawei Web Categorization.; https://isecurity.huawei.com/; sec at huawei dot com)'
+  bot:
+    name: HuaweiWebCatBot
+    category: Crawler
+    url: https://isecurity.huawei.com
+    producer:
+      name: Huawei Technologies Co., Ltd.
+      url: https://huawei.com
+-
+  user_agent: Mozilla/5.0 (compatible; SputnikFaviconBot/1.2; +http://corp.sputnik.ru/webmaster)
+  bot:
+    name: Sputnik Favicon Bot
+    url: ""
+    category: Crawler
+-
+  user_agent: Hatena-Favicon2 (http://www.hatena.ne.jp/faq/)
+  bot:
+    name: Hatena Favicon
+    category: Crawler
+    url: https://www.hatena.ne.jp/faq/
+    producer:
+      name: Hatena Co., Ltd.
+      url: https://www.hatena.ne.jp
+-
+  user_agent: Mozilla/5.0 (compatible; RyowlEngine/1.0; +https://ryowl.org)
+  bot:
+    name: Ryowl
+    category: Crawler
+    url: https://ryowl.org
+-
+  user_agent: Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)
+  bot:
+    name: Odnoklassniki Bot
+    category: Crawler
+    url: https://odnoklassniki.ru
+-
+  user_agent: Mediatoolkitbot (complaints@mediatoolkit.com)
+  bot:
+    name: Mediatoolkit Bot
+    category: Crawler
+    url: https://mediatoolkit.com
+-
+  user_agent: ZoominfoBot (zoominfobot at zoominfo dot com)
+  bot:
+    name: ZoominfoBot
+    category: Crawler
+    url: https://www.zoominfo.com
+-
+  user_agent: Mozilla/5.0 (compatible; WeViKaBot/1.0; +http://www.wevika.de/)
+  bot:
+    name: WeViKaBot
+    category: Crawler
+    url: http://www.wevika.de
+-
+  user_agent: Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)
+  bot:
+    name: SEOkicks
+    category: Crawler
+    url: https://www.seokicks.de/robot.html
+-
+  user_agent: Mozilla/5.0 (compatible; Plukkie/1.6; http://www.botje.com/plukkie.htm)
+  bot:
+    name: Plukkie
+    category: Crawler
+    url: http://www.botje.com/plukkie.htm
+-
+  user_agent: Mozilla/5.0 (compatible; proximic; https://www.comscore.com/Web-Crawler)
+  bot:
+    name: Comscore
+    category: Crawler
+    url: https://www.comscore.com/Web-Crawler
+-
+  user_agent: Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html)
+  bot:
+    name: SurdotlyBot
+    category: Crawler
+    url: http://sur.ly/bot.html
+-
+  user_agent: Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)
+  bot:
+    name: Gowikibot
+    category: Crawler
+    url: http:/www.gowikibot.com
+-
+  user_agent: Mozilla/5.0 (compatible; SabsimBot/3.0; +https://sabsim.com)
+  bot:
+    name: SabsimBot
+    category: Crawler
+    url: https://sabsim.com
+-
+  user_agent: Mozilla/5.0 (compatible; LumtelBot/1.0; +http://lumtel.com)
+  bot:
+    name: LumtelBot
+    category: Crawler
+    url: https://umtel.com
+-
+  user_agent: Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)
+  bot:
+    name: PiplBot
+    category: Crawler
+    url: http://www.pipl.com/bot
+-
+  user_agent: Mozilla/5.0 (compatible; woobot/2.0; +https://www.woorank.com/bot)
+  bot:
+    name: WooRank
+    category: Crawler
+    url: https://www.woorank.com/bot
+-
+  user_agent: YahooMailProxy; https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html
+  bot:
+    name: Yahoo! Mail Proxy
+    category: Service Agent
+    url: https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html
+    producer:
+      name: Yahoo! Inc.
+      url: http://www.yahoo.com
+-
+  user_agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Cookiebot/1.0;  http://cookiebot.com/) Chrome/79.0.3945.117 Safari/537.36'
+  bot:
+    name: Cookiebot
+    category: Crawler
+    url: https://support.cookiebot.com/hc/en-us/articles/360014264140-Scanner-User-Agent
+    producer:
+      name: Cybot A/S
+      url: https://www.cybot.com/
+-
+  user_agent: NetSystemsResearch studies the availability of various services across the internet. Our website is netsystemsresearch.com
+  bot:
+    name: NetSystemsResearch
+    category: Security Checker
+    url: https://www.netsystemsresearch.com/
+    producer:
+      name: NET SYSTEMS RESEARCH LLC
+      url: https://www.netsystemsresearch.com/
+-
+  user_agent: 'Mozilla/5.0 (compatible; CensysInspect/1.1;  https://about.censys.io/)'
+  bot:
+    name: CensysInspect
+    category: Security Checker
+    url: https://about.censys.io/
+    producer:
+      name: Censys, Inc.
+      url: https://censys.io/
+-
+  user_agent: https://gdnplus.com:Gather Analyze Provide.
+  bot:
+    name: GDNP
+    category: Crawler
+    url: https://gdnplus.com/
+    producer:
+      name: Global Digital Network Plus, LLC
+      url: https://gdnplus.com/
+-
+  user_agent: WellKnownBot/0.1 (https://well-known.dev)
+  bot:
+    name: WellKnownBot
+    category: Crawler
+    url: https://well-known.dev
+-
+  user_agent: Mozilla/5.0 (compatible; Adsbot/3.1;  https://seostar.co/robot/)
+  bot:
+    name: Adsbot
+    category: Crawler
+    url: https://seostar.co/robot/
+-
+  user_agent: MTRobot/0.2 (Metrics Tools Analytics Crawler; https://metrics-tools.de/robot.html; crawler@metrics-tools.de)
+  bot:
+    name: MTRobot
+    category: Crawler
+    url: https://metrics-tools.de/robot.html
+    producer:
+      name: Metrics Tools
+      url: https://metrics-tools.de/
+-
+  user_agent: My User Agent 1.0
+  bot:
+    name: Generic Bot
+-
+  user_agent: serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)
+  bot:
+    name: serpstatbot
+    category: Crawler
+    url: http://serpstatbot.com/
+    producer:
+      name: Netpeak Ltd
+      url: https://netpeak.net/
+-
+  user_agent: colly - https://github.com/gocolly/colly/v2
+  bot:
+    name: colly
+    category: Crawler
+    url: https://github.com/gocolly/colly/
+-
+  user_agent: l9tcpid/v1.0.0
+  bot:
+    name: l9tcpid
+    category: Security Checker
+    url: https://github.com/LeakIX/l9tcpid
+-
+  user_agent: Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)
+  bot:
+    name: MegaIndex
+    category: Crawler
+    url: https://megaindex.com/crawler
+-
+  user_agent: Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)
+  bot:
+    name: Seekport
+    category: Crawler
+    url: http://www.seekport.com/
+    producer:
+      name: SISTRIX GmbH
+      url: https://www.sistrix.de/
+-
+  user_agent: Mozilla/5.0 (compatible; seolyt/1.1; +https://seolyt.com)
+  bot:
+    name: seolyt
+    category: Crawler
+    url: https://seolyt.com/
+-
+  user_agent: Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)
+  bot:
+    name: YaK
+    category: Crawler
+    url: https://www.linkfluence.com/
+    producer:
+      name: Linkfluence SAS
+      url: https://www.linkfluence.com/
+-
+  user_agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0
+  bot:
+    name: KomodiaBot
+    category: Crawler
+    url: http://www.komodia.com/newwiki/index.php/URL_server_crawler
+    producer:
+      name: Komodia Inc.
+      url: https://www.komodia.com/
+-
+  user_agent: Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; Catchpoint) like Gecko
+  bot:
+    name: Catchpoint
+    category: Site Monitor
+    url: https://www.catchpoint.com/
+    producer:
+      name: Catchpoint Systems
+      url: https://www.catchpoint.com/
+-
+  user_agent: Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36
+  bot:
+    name: Better Uptime Bot
+    category: Site Monitor
+    url: https://betteruptime.com/faq
+    producer:
+      name: Better Uptime
+      url: https://betteruptime.com/
+-
+  user_agent: gobuster/3.1.0
+  bot:
+    name: Gobuster
+    url: https://github.com/OJ/gobuster
+-
+  user_agent: k6/0.31.1 (https://k6.io/)
+  bot:
+    name: K6
+    url: https://k6.io/
+-
+  user_agent: Mozilla/5.0 (compatible; Neevabot/1.0;  https://neeva.com/neevabot)
+  bot:
+    name: Neevabot
+    category: Search bot
+    url: https://neeva.com/neevabot
+    producer:
+      name: Neeva Inc.
+      url: https://neeva.com/
+-
+  user_agent: LinkPreview/1.5 (http://linkpreview.net)
+  bot:
+    name: LinkPreview
+    category: Service Agent
+    url: https://www.linkpreview.net/
+-
+  user_agent: Mozilla/5.0 (compatible; JungleKeyThumbnail/1.1; +http://www.junglekey.fr/)
+  bot:
+    name: JungleKeyThumbnail
+    category: Crawler
+    url: https://junglekey.com/
+-
+  user_agent: rocketmonitor 1.0
+  bot:
+    name: RocketMonitorBot
+    category: Site Monitor
+    url: https://www.radiomast.io/docs/stream-monitoring/technical_details.html
+    producer:
+      name: Radio Mast, Inc.
+      url: https://www.radiomast.io/
+-
+  user_agent: RocketMonitorBot/1.0 ( https://www.radiomast.io)
+  bot:
+    name: RocketMonitorBot
+    category: Site Monitor
+    url: https://www.radiomast.io/docs/stream-monitoring/technical_details.html
+    producer:
+      name: Radio Mast, Inc.
+      url: https://www.radiomast.io/
+-
+  user_agent: SitemapParser-VIPnytt/1.1 ( https://github.com/VIPnytt/SitemapParser/blob/master/README.md)
+  bot:
+    name: SitemapParser-VIPnytt
+    category: Crawler
+    url: https://github.com/VIPnytt/SitemapParser/
+-
+  user_agent: Turnitin (https://bit.ly/2UvnfoQ)
+  bot:
+    name: Turnitin
+    category: Crawler
+    url: https://turnitin.com/robot/crawlerinfo.html
+-
+  user_agent: Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36
+  bot:
+    name: Google StoreBot
+    category: Crawler
+-
+  user_agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.56 Safari/537.36 DMBrowser/2.1 (UV)
+  bot:
+    name: Dotcom Monitor
+    category: Site Monitor
+    url: https://www.dotcom-monitor.com
+-
+  user_agent: Mozilla/5.0 (Windows NT 6.2; WOW64; rv:29.0) Gecko/20100101 /29.0 DMBrowser-BV
+  bot:
+    name: Dotcom Monitor
+    category: Site Monitor
+    url: https://www.dotcom-monitor.com
+-
+  user_agent: Mozilla/5.0 (compatible; ThinkChaos/0.3.0; +In_the_test_phase,_if_the_ThinkChaos_brings_you_trouble,_please_add_disallow_to_the_robots.txt._Thank_you.)
+  bot:
+    name: ThinkChaos
+    category: Crawler
+-
+  user_agent: Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)
+  bot:
+    name: DataForSeoBot
+    category: Crawler
+    url: https://dataforseo.com/dataforseo-bot