device_detector 0.9.1 → 1.0.4

data/spec/fixtures/client/mobile_app.yml

@@ -1,30 +1,217 @@
--
+---
+- 
   user_agent: Pulse/4.0.5 (iPhone; iOS 7.0.6; Scale/2.00)
   client:
     type: mobile app
     name: Pulse
     version: "4.0.5"
--
+- 
   user_agent: Mozilla/5.0 (Linux; Android 4.4.4; SM-G7509 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 MicroMessenger/6.0.2.56_r958800.520 NetType/WIFI
   client:
     type: mobile app
     name: WeChat
     version: "6.0.2.56.r958800.520"
--
+- 
   user_agent: Mozilla/5.0 (Linux; U; Android 4.0.4; en-us; ZTE U807N Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 Weibo (ZTE-ZTE U807N__weibo__5.0.0__android__android4.0.4)
   client:
     type: mobile app
     name: Sina Weibo
     version: "5.0.0"
--
+- 
   user_agent: FeedR/1.0 (Android)
   client:
     type: mobile app
     name: FeedR
     version: "1.0"
--
+- 
   user_agent: Pinterest/3.3.3 CFNetwork/609.1.4 Darwin/13.0.0
   client:
     type: mobile app
     name: Pinterest
-    version: "3.3.3"
+    version: "3.3.3"
+- 
+  user_agent: Instacast/5.0a8 CFNetwork/711.1.11 Darwin/13.4.0
+  client:
+    type: mobile app
+    name: Instacast
+    version: "5.0a8"
+- 
+  user_agent: InstacastHD/1.1.2 CFNetwork/711.0.6 Darwin/14.0.0
+  client:
+    type: mobile app
+    name: Instacast
+    version: "1.1.2"
+- 
+  user_agent: Podcasts/2.1.2
+  client:
+    type: mobile app
+    name: Podcasts
+    version: "2.1.2"
+- 
+  user_agent: Shifty Jelly Pocket Casts, Android v4.4.3.1
+  client:
+    type: mobile app
+    name: Pocket Casts
+    version: "4.4.3.1"
+- 
+  user_agent: Pocket Casts
+  client:
+    type: mobile app
+    name: Pocket Casts
+    version: ""
+- 
+  user_agent: Mozilla/5.0 (Linux; U; en-us; BeyondPod)
+  client:
+    type: mobile app
+    name: BeyondPod
+    version: ""
+- 
+  user_agent: AntennaPod/0.9.9.1
+  client:
+    type: mobile app
+    name: AntennaPod
+    version: "0.9.9.1"
+- 
+  user_agent: AntennaPod/
+  client:
+    type: mobile app
+    name: AntennaPod
+    version: ""
+- 
+  user_agent: Overcast/1.0 (+http://overcast.fm/; iOS podcast app)
+  client:
+    type: mobile app
+    name: Overcast
+    version: "1.0"
+- 
+  user_agent: Podkicker/1.9.4
+  client:
+    type: mobile app
+    name: Podkicker
+    version: "1.9.4"
+- 
+  user_agent: Podkicker Pro/1.9.4
+  client:
+    type: mobile app
+    name: Podkicker
+    version: "1.9.4"
+- 
+  user_agent: Castro/64 CFNetwork/672.1.15 Darwin/14.0.0
+  client:
+    type: mobile app
+    name: Castro
+    version: "64"
+- 
+  user_agent: Mozilla/5.0 (Linux; U; Windows NT 6.1; en-us; dream) DoggCatcher
+  client:
+    type: mobile app
+    name: DoggCatcher
+    version: ""
+- 
+  user_agent: Mozilla/5.0 (Linux; U; Android 4.2.2; ja-jp; SOL22 Build/10.3.1.D.0.257) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 YJApp-ANDROID jp.co.yahoo.android.yjtop/3.15.1
+  client:
+    type: mobile app
+    name: Yahoo! Japan
+    version: "3.15.1"
+- 
+  user_agent: RSSRadio/9220 (iPad;iOS;12.3.1)
+  client:
+    type: mobile app
+    name: RSSRadio
+    version: "9220"
+- 
+  user_agent: RSSRadio/9252
+  client:
+    type: mobile app
+    name: RSSRadio
+    version: "9252"
+- 
+  user_agent: RSSRadio/
+  client:
+    type: mobile app
+    name: RSSRadio
+    version: ""
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 7.0; JMM-AL00 Build/HONORJMM-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 SogouSearch Android1.0 version3.0 AppVersion/6002
+  client:
+    type: mobile app
+    name: SogouSearch App
+    version: "3.0"
+- 
+  user_agent: Dalvik/2.1.0 (Linux; U; Android 6.0.1; MI 4LTE MIUI/V7.3.2.0.MXDCNDD) NewsArticle/5.1.3
+  client:
+    type: mobile app
+    name: NewsArticle App
+    version: "5.1.3"
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 5.1; m2 note) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 baidubrowser/5.3.4.0 (Baidu; P1 4.3.1) tieba/6.7.2 BMW
+  client:
+    type: mobile app
+    name: tieba
+    version: "6.7.2"
+- 
+  user_agent: com.douban.group/2.2.0(220) Mozilla/5.0 (Linux; U; Android 4.1.1; zh-cn; MI 2A Build/JRO03L) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30
+  client:
+    type: mobile app
+    name: douban App
+    version: "2.2.0"
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 7.0; VIE-L09 Build/HUAWEIVIE-L09; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/59.0.3071.125 Mobile Safari/537.36 BingWeb/6.9.25207603
+  client:
+    type: mobile app
+    name: BingWebApp
+    version: "6.9.25207603"
+- 
+  user_agent: Mozilla/5.0 (Linux; U; Android 4.1.2; fr-ca; LT30a Build/9.1.A.0.489) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 GSA/3.3.11.1069658.arm
+  client:
+    type: mobile app
+    name: Google Search App
+    version: "3.3.11.1069658"
+- 
+  user_agent: Mozilla/5.0 (Linux; U; Android 2.3.4; ru-ru; WT19i Build/4.0.2.A.0.58) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 Flipboard/2.1.2/854,2.1.2.854,2013-12-05 11:59, +0300
+  client:
+    type: mobile app
+    name: Flipboard App
+    version: "2.1.2"
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; ASUS_Z011D Build/MMB29P; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36 Instagram 53.0.0.13.84 Android (23/6.0.1; 480dpi; 1080x1920; asus; ASUS_Z011D; ASUS_Z011; qcom; fr_FR; 116756948)
+  client:
+    type: mobile app
+    name: Instagram App
+    version: "53.0.0.13.84"
+- 
+  user_agent: TCL J706T_TD/1.0 Linux/3.4.5 Android/4.1.2 Release/08.27.2013 Browser/AppleWebKit534.30 Profile/MIDP-2.0 Configuration/CLDC-1.1 baiduboxapp/4.2 (Baidu; P1 4.1.2)
+  client:
+    type: mobile app
+    name: Baidu Box App
+    version: "4.2"
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 4.4.4; GT-I9195I Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Crosswalk/14.43.343.17 Mobile Safari/537.36
+  client:
+    type: mobile app
+    name: CrosswalkApp
+    version: "14.43.343.17"
+-
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.2.10 (KHTML, like Gecko) Mobile/15A372 Twitter for iPhone/7.52
+  client:
+    type: mobile app
+    name: Twitter
+    version: "7.52"
+-
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X; en-us) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone
+  client:
+    type: mobile app
+    name: Twitter
+    version: ""
+-
+  user_agent: Mozilla/5.0 (Linux; Android 5.1; HTC Desire 628 Build/LMY47D; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.108 Mobile Safari/537.36 JsSdk/2 TopBuzz/10.2.1 NetType/3G
+  client:
+    type: mobile app
+    name: TopBuzz
+    version: 10.2.1
+-
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Mobile/15E148 Snapchat/10.77.0.54 (like Safari/604.1)
+  client:
+    type: mobile app
+    name: Snapchat
+    version: 10.77.0.54

data/spec/fixtures/client/pim.yml

@@ -1,96 +1,115 @@
--
+---
+- 
   user_agent: Outlook-Express/7.0 (MSIE 7.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; AskTbORJ/5.15.9.29495; .NET4.0E; TmstmpExt)
   client:
     type: pim
     name: Outlook Express
     version: "7.0"
--
+- 
   user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Win64; x64; Trident/7.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3; .NET CLR 1.1.4322; FDM; Tablet PC 2.0; .NET4.0E; Microsoft Outlook 14.0.7113; ms-office; MSOffice 14)
   client:
     type: pim
     name: Microsoft Outlook
     version: "14.0.7113"
--
+- 
   user_agent: Mozilla/5.0 (X11; Linux i686; rv:17.0) Gecko/20130330 Thunderbird/17.0.5
   client:
     type: pim
     name: Thunderbird
     version: "17.0.5"
--
+- 
   user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Thunderbird/24.4.0 Lightning/2.6.4
   client:
     type: pim
     name: Thunderbird
     version: "24.4.0"
--
+- 
   user_agent: Airmail 1.4 rv:238 (Macintosh; Mac OS X 10.9.2; hr_HR)
   client:
     type: pim
     name: Airmail
     version: "1.4"
--
+- 
   user_agent: Airmail 1.3.3 rv:237 (Macintosh; Mac OS X 10.9.2; en_US)
   client:
     type: pim
     name: Airmail
     version: "1.3.3"
--
+- 
   user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Icedove/24.4.0
   client:
     type: pim
     name: Thunderbird
     version: "24.4.0"
--
+- 
   user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Icedove/24.4.0 Lightning/2.6.5
   client:
     type: pim
     name: Thunderbird
     version: "24.4.0"
--
+- 
   user_agent: Mozilla/4.0 (compatible; Lotus-Notes/6.0; Windows-NT)
   client:
     type: pim
     name: Lotus Notes
     version: "6.0"
--
+- 
   user_agent: Barca/2.8.4400
   client:
     type: pim
     name: Barca
     version: "2.8.4400"
--
+- 
   user_agent: BarcaPro/1.4 L.1001
   client:
     type: pim
     name: Barca
     version: "1.4"
--
+- 
   user_agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.8) Gecko/20100317 Postbox/1.1.3
   client:
     type: pim
     name: Postbox
     version: "1.1.3"
--
+- 
   user_agent: Postbox 1.0b14 (Windows/2009072715)
   client:
     type: pim
     name: Postbox
     version: "1.0"
--
+- 
   user_agent: MailBar/1.3.2 (Mac OS X Version 10.11.1 (Build 15B42))
   client:
     type: pim
     name: MailBar
-    version: 1.3.2
--
+    version: "1.3.2"
+- 
   user_agent: The Bat! 4.0.0.22
   client:
     type: pim
     name: The Bat!
     version: "4.0.0.22"
--
+- 
   user_agent: The Bat! Voyager 4.0.18.4
   client:
     type: pim
     name: The Bat!
-    version: "4.0.18.4"
+    version: "4.0.18.4"
+- 
+  user_agent: DAVdroid/1.6.2-ose (2017/06/23; dav4android; okhttp3) Android/7.0
+  client:
+    type: pim
+    name: DAVdroid
+    version: "1.6.2"
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:10.0.12) Gecko/20130823 Firefox/10.0.11esrpre Iceape/2.7.12
+  client:
+    type: pim
+    name: SeaMonkey
+    version: "2.7.12"
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:29.0) Gecko/20100101 Firefox/29.0 SeaMonkey/2.26a1 Lightning/3.1a1
+  client:
+    type: pim
+    name: SeaMonkey
+    version: "2.26"

data/spec/fixtures/detector/bots.yml

@@ -7,7 +7,7 @@
     url: http://www.so.com/help/help_3_2.html
     producer:
       name: Online Media Group, Inc.
-      url:
+      url: ""
 - 
   user_agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MDDR; .NET4.0C; .NET4.0E; .NET CLR 1.1.4322; Tablet PC 2.0); 360Spider
   bot:
@@ -16,7 +16,7 @@
     url: http://www.so.com/help/help_3_2.html
     producer:
       name: Online Media Group, Inc.
-      url:
+      url: ""
 - 
   user_agent: Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider
   bot:
@@ -25,7 +25,7 @@
     url: http://www.so.com/help/help_3_2.html
     producer:
       name: Online Media Group, Inc.
-      url:
+      url: ""
 - 
   user_agent: Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider
   bot:
@@ -34,7 +34,7 @@
     url: http://www.so.com/help/help_3_2.html
     producer:
       name: Online Media Group, Inc.
-      url:
+      url: ""
 - 
   user_agent: 'ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com'
   bot:
@@ -62,7 +62,7 @@
   bot:
     name: AddThis.com
     category: Social Media Agent
-    url:
+    url: ""
     producer:
       name: Clearspring Technologies, Inc.
       url: http://www.clearspring.com
@@ -75,6 +75,31 @@
     producer:
       name: Alexa Internet
       url: http://www.alexa.com
+- 
+  user_agent: Mozilla/5.0 (compatible; alexa site audit/1.0; http://www.alexa.com/help/webmasters; )
+  bot:
+    name: Alexa Site Audit
+    category: Site Monitor
+    url: http://www.alexa.com/help/webmasters
+    producer:
+      name: Alexa Internet
+      url: http://www.alexa.com
+- 
+  user_agent: Amazon Route 53 Health Check Service; ref:xxxxxxxx-xxxx-xxxxxxxxx-xxxxxxxxxxxx; report http://amzn.to/xxxxxxx
+  bot:
+    name: Amazon Route53 Health Check
+    category: Service Agent
+    producer:
+      name: Amazon Web Services
+      url: https://aws.amazon.com/
+- 
+  user_agent: Amazon-Route53-Health-Check-Service (ref b0eb04d5-cb5e-40e7-839b-558e52fc3f0d; report http://amzn.to/1vsZADi)
+  bot:
+    name: Amazon Route53 Health Check
+    category: Service Agent
+    producer:
+      name: Amazon Web Services
+      url: https://aws.amazon.com/
 - 
   user_agent: AmorankSpider/0.1; +http://amorank.com/webcrawler.html
   bot:
@@ -93,6 +118,15 @@
     producer:
       name: Analytics SEO
       url: http://www.analyticsseo.com
+- 
+  user_agent: ApacheBench/2.3
+  bot:
+    name: ApacheBench
+    category: Benchmark
+    url: https://httpd.apache.org/docs/2.4/programs/ab.html
+    producer:
+      name: The Apache Software Foundation
+      url: http://www.apache.org/foundation/
 - 
   user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10 _1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)
   bot:
@@ -111,15 +145,60 @@
     producer:
       name: Apple Inc
       url: http://www.apple.com
+- 
+  user_agent: Arachni/v1.5.1
+  bot:
+    name: Arachni
+    category: Security Checker
+    url: http://www.arachni-scanner.com
+    producer:
+      name: Sarosys LLC
+      url: http://www.sarosys.com/
 - 
   user_agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
   bot:
     name: Ask Jeeves
     category: Search bot
-    url:
+    url: ""
     producer:
       name: Ask Jeeves Inc.
       url: http://www.ask.com
+- 
+  user_agent: Mozilla/5.0 (compatible;AspiegelBot)
+  bot:
+    name: AspiegelBot
+    category: Crawler
+    url: https://aspiegel.com/
+    producer:
+      name: Huawei
+      url: https://www.huawei.com/
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)
+  bot:
+    name: AspiegelBot
+    category: Crawler
+    url: https://aspiegel.com/
+    producer:
+      name: Huawei
+      url: https://www.huawei.com/
+- 
+  user_agent: AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)
+  bot:
+    name: Awario
+    category: Feed Fetcher
+    url: https://awario.com/bots.html
+    producer:
+      name: Awario
+      url: https://awario.com/
+- 
+  user_agent: AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)
+  bot:
+    name: Awario
+    category: Search bot
+    url: https://awario.com/bots.html
+    producer:
+      name: Awario
+      url: https://awario.com/
 - 
   user_agent: Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
   bot:
@@ -136,12 +215,12 @@
     category: Crawler
     url: http://law.di.unimi.it/BUbiNG.html
     producer:
-      name:
-      url:
+      name: The Laboratory for Web Algorithmics (LAW)
+      url: 'http://law.di.unimi.it/software.php#buging'
 - 
-  user_agent: Backlink-Ceck.de (+http://www.backlink-check.de/bot.html)
+  user_agent: Backlink-Check.de (+http://www.backlink-check.de/bot.html)
   bot:
-    name: Backlink-Ceck.de
+    name: Backlink-Check.de
     category: Crawler
     url: http://www.backlink-check.de/bot.html
     producer:
@@ -226,8 +305,8 @@
     url: https://bazqux.com/fetcher
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
   bot:
@@ -300,6 +379,15 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
+- 
+  user_agent: bitlybot/3.0
+  bot:
+    name: BitlyBot
+    category: Crawler
+    url: https://bitly.com
+    producer:
+      name: Bitly, Inc.
+      url: https://bitly.com
 - 
   user_agent: Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)
   bot:
@@ -316,17 +404,34 @@
     url: http://www.bloglovin.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Blogtrottr/2.0
   bot:
     name: Blogtrottr
-    url:
+    url: ""
     category: Feed Fetcher
     producer:
       name: Blogtrottr Ltd
       url: https://blogtrottr.com/
+- 
+  user_agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com
+  bot:
+    name: BoardReader
+    category: Search bot
+    url: http://boardreader.com/
+    producer:
+      name: Effyis Inc
+      url: http://boardreader.com/
+- 
+  user_agent: BoardReader Blog Indexer(http://boardreader.com)
+  bot:
+    name: BoardReader Blog Indexer
+    category: Crawler
+    producer:
+      name: BoardReader
+      url: http://boardreader.com/
 - 
   user_agent: Mozilla/5.0 (compatible; BountiiBot/1.1; +http://bountii.com/contact.php)
   bot:
@@ -336,6 +441,15 @@
     producer:
       name: Bountii Inc.
       url: http://bountii.com
+- 
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)
+  bot:
+    name: BrandVerity
+    category: Crawler
+    url: https://www.brandverity.com/why-is-brandverity-visiting-me
+    producer:
+      name: BrandVerity, Inc.
+      url: https://www.brandverity.com/
 - 
   user_agent: Browsershots
   bot:
@@ -345,6 +459,15 @@
     producer:
       name: Browsershots.org
       url: http://browsershots.org
+- 
+  user_agent: Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)
+  bot:
+    name: Buck
+    category: Search bot
+    url: https://hypefactors.com/
+    producer:
+      name: Hypefactors A/S
+      url: https://hypefactors.com/
 - 
   user_agent: Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
   bot:
@@ -363,6 +486,24 @@
     producer:
       name: Topsy Labs
       url: http://labs.topsy.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.5668.1446 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com
+  bot:
+    name: Bytespider
+    category: Search bot
+    url: https://bytedance.com/
+    producer:
+      name: ByteDance Ltd.
+      url: https://bytedance.com/
+- 
+  user_agent: CATExplorador/1.0beta (sistemes at domini dot cat; http://domini.cat/catexplorador.html)
+  bot:
+    name: CATExplorador
+    category: Search bot
+    url: https://fundacio.cat/ca/domini/
+    producer:
+      name: Fundació puntCAT
+      url: https://fundacio.cat/ca/domini/
 - 
   user_agent: Mozilla/5.0 (compatible; CareerBot/1.1; +http://www.career-x.de/bot.html)
   bot:
@@ -372,6 +513,15 @@
     producer:
       name: career-x GmbH
       url: http://www.career-x.de
+- 
+  user_agent: Castro 2, Episode Duration Lookup
+  bot:
+    name: Castro 2
+    category: Service Agent
+    url: http://supertop.co/castro/
+    producer:
+      name: Supertop
+      url: http://supertop.co
 - 
   user_agent: Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)
   bot:
@@ -381,6 +531,15 @@
     producer:
       name: 10betterpages GmbH
       url: http://cliqz.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Cloudflare-AMP/1.0; +https://amp.cloudflare.com/doc/fetcher.html) AppleWebKit/534.34
+  bot:
+    name: CloudFlare AMP Fetcher
+    category: Crawler
+    url: https://amp.cloudflare.com/doc/fetcher.html
+    producer:
+      name: CloudFlare
+      url: http://www.cloudflare.com
 - 
   user_agent: Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34
   bot:
@@ -390,6 +549,15 @@
     producer:
       name: CloudFlare
       url: http://www.cloudflare.com
+- 
+  user_agent: collectd/5.5.1
+  bot:
+    name: Collectd
+    url: https://collectd.org/
+    category: Site Monitor
+    producer:
+      name: Collectd
+      url: https://collectd.org/
 - 
   user_agent: CommaFeed/1.0 (http://www.commafeed.com)
   bot:
@@ -397,17 +565,107 @@
     url: http://www.commafeed.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
+- 
+  user_agent: coccocbot-web/1.0 (+http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
 - 
   user_agent: Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)
   bot:
     name: Cốc Cốc Bot
-    url: http://help.coccoc.com/
     category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
     producer:
       name: Cốc Cốc
-      url: http://coccoc.com/
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-ads/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-fast/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-shopping/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Datadog Agent/5.10.1
+  bot:
+    name: Datadog Agent
+    url: https://github.com/DataDog/dd-agent
+    category: Site Monitor
+    producer:
+      name: Datadog
+      url: https://www.datadoghq.com/
+- 
+  user_agent: Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36
+  bot:
+    name: Datanyze
+    url: ""
+    category: Crawler
+    producer:
+      name: Datanyze
+      url: https://www.datanyze.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Dataprovider/6.92; +https://www.dataprovider.com/)
+  bot:
+    name: Dataprovider
+    category: Crawler
+    url: ""
+    producer:
+      name: Dataprovider B.V.
+      url: https://www.dataprovider.com/
 - 
   user_agent: Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0;
   bot:
@@ -431,7 +689,7 @@
   bot:
     name: Dazoobot
     category: Search bot
-    url:
+    url: ""
     producer:
       name: DAZOO.FR
       url: http://dazoo.fr
@@ -449,7 +707,7 @@
   bot:
     name: Domain Re-Animator Bot
     category: Crawler
-    url:
+    url: ""
     producer:
       name: Domain Re-Animator, LLC
       url: http://domainreanimator.com
@@ -458,7 +716,7 @@
   bot:
     name: Domain Re-Animator Bot
     category: Crawler
-    url:
+    url: ""
     producer:
       name: Domain Re-Animator, LLC
       url: http://domainreanimator.com
@@ -471,15 +729,33 @@
     producer:
       name: SEOmoz, Inc.
       url: http://moz.com/
+- 
+  user_agent: DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)
+  bot:
+    name: DuckDuckGo Bot
+    category: Search bot
+    url: https://duckduckgo.com/duckduckbot
+    producer:
+      name: DuckDuckGo
+      url: https://duckduckgo.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)
+  bot:
+    name: DuckDuckGo Bot
+    category: Search bot
+    url: https://duckduckgo.com/duckduckbot
+    producer:
+      name: DuckDuckGo
+      url: https://duckduckgo.com/
 - 
   user_agent: EMail Exractor
   bot:
     name: EMail Exractor
     category: Crawler
-    url:
+    url: ""
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)
   bot:
@@ -498,6 +774,15 @@
     producer:
       name: easou ICP
       url: http://www.easou.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)
+  bot:
+    name: Embedly
+    category: Crawler
+    url: https://support.embed.ly/hc/en-us
+    producer:
+      name: A Medium, Corp.
+      url: https://medium.com/
 - 
   user_agent: Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)
   bot:
@@ -539,7 +824,7 @@
   bot:
     name: Ezooms
     category: Crawler
-    url:
+    url: ""
     producer:
       name: SEOmoz, Inc.
       url: http://moz.com/
@@ -552,6 +837,15 @@
     producer:
       name: Facebook
       url: http://www.facebook.com
+- 
+  user_agent: facebookexternalua
+  bot:
+    name: Facebook External Hit
+    category: Social Media Agent
+    url: https://www.facebook.com/externalhit_uatext.php
+    producer:
+      name: Facebook
+      url: http://www.facebook.com
 - 
   user_agent: facebookplatform/1.0 (+http://developers.facebook.com)
   bot:
@@ -577,8 +871,8 @@
     url: http://www.feedburner.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: 'Feedbin - 9 subscribers'
   bot:
@@ -586,8 +880,8 @@
     url: http://feedbin.com/
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)
   bot:
@@ -595,8 +889,8 @@
     url: http://www.feedly.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: FeedlyApp/1.0 (http://www.feedly.com)
   bot:
@@ -604,8 +898,8 @@
     url: http://www.feedly.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: FeedlyBot/1.0 (http://feedly.com)
   bot:
@@ -613,8 +907,8 @@
     url: http://www.feedly.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: MetaFeedly/1.0 (http://www.feedly.com)
   bot:
@@ -622,8 +916,8 @@
     url: http://www.feedly.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Feedspot http://www.feedspot.com
   bot:
@@ -631,8 +925,8 @@
     url: http://www.feedspot.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)
   bot:
@@ -640,25 +934,119 @@
     url: http://feedafever.com/
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
+- 
+  user_agent: Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)
+  bot:
+    name: Findxbot
+    category: Crawler
+    url: http://www.findxbot.com
+- 
+  user_agent: Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)
+  bot:
+    name: Flipboard
+    url: http://flipboard.com/browserproxy
+    category: Feed Fetcher
+    producer:
+      name: Flipboard
+      url: http://flipboard.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; FlipboardRSS/1.2; +http://flipboard.com/browserproxy)
+  bot:
+    name: Flipboard
+    url: http://flipboard.com/browserproxy
+    category: Feed Fetcher
+    producer:
+      name: Flipboard
+      url: http://flipboard.com/
+- 
+  user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
+  bot:
+    name: FreshRSS
+    category: Feed Fetcher
+    url: https://freshrss.org/
+- 
+  user_agent: FreshRSS/1.12.0 (Linux; https://freshrss.org)
+  bot:
+    name: FreshRSS
+    category: Feed Fetcher
+    url: https://freshrss.org/
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64; GTmetrix https://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36
+  bot:
+    name: GTmetrix
+    category: Crawler
+    url: https://gtmetrix.com/
+    producer:
+      name: Carbon60 Operating Co. Ltd.
+      url: https://www.carbon60.com/
+- 
+  user_agent: 7Siters/1.07 (+https://7ooo.ru/siters/)
+  bot:
+    name: Generic Bot
+- 
+  user_agent: COMODO SSL Checker
+  bot:
+    name: Generic Bot
+- 
+  user_agent: help@dataminr.com
+  bot:
+    name: Generic Bot
+- 
+  user_agent: 'HybridBot (hybrid.ru/about. If our bot caused problems please contact us. Contact email: m.lyashkov@targetix.net)'
+  bot:
+    name: Generic Bot
+- 
+  user_agent: 'Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)'
+  bot:
+    name: Generic Bot
+- 
+  user_agent: Mozilla/5.0 (compatible; +http://tweetedtimes.com)
+  bot:
+    name: Generic Bot
+- 
+  user_agent: Mozilla/5.0 (compatible; Shareaholicbot/2.0; +http://www.shareaholic.com/bot)
+  bot:
+    name: Generic Bot
+- 
+  user_agent: Mozilla/5.0 (compatible; TrendsmapResolver/0.1)
+  bot:
+    name: Generic Bot
 - 
   user_agent: niki-bot
   bot:
     name: Generic Bot
 - 
-  user_agent: StatusCake Error Detector Crawler V0.01
+  user_agent: PHPCrawl
+  bot:
+    name: Generic Bot
+- 
+  user_agent: robots
+  bot:
+    name: Generic Bot
+- 
+  user_agent: SeopultContentAnalyzer/1.0
   bot:
     name: Generic Bot
 - 
   user_agent: Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)
   bot:
     name: Genieo Web filter
-    category:
+    category: ""
     url: http://www.genieo.com/webfilter.html
     producer:
       name: Genieo
       url: http://www.genieo.com
+- 
+  user_agent: GigablastOpenSource/1.0
+  bot:
+    name: Gigablast
+    category: Search bot
+    url: https://github.com/gigablast/open-source-search-engine
+    producer:
+      name: Matt Wells
+      url: http://www.gigablast.com/faq.html
 - 
   user_agent: Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)
   bot:
@@ -666,14 +1054,14 @@
     category: Crawler
     url: http://glutenfreepleasure.com/
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)
   bot:
     name: Gmail Image Proxy
     category: Crawler
-    url:
+    url: ""
     producer:
       name: Google Inc.
       url: http://www.google.com
@@ -686,6 +1074,11 @@
     producer:
       name: NTT Resonant
       url: http://goo.ne.jp
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon
+  bot:
+    name: Google Favicon
+    category: Crawler
 - 
   user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0_1 like Mac OS X; en-us) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Version/4.0.5 Mobile/8A306 Safari/6531.22.7
   bot:
@@ -700,7 +1093,25 @@
   bot:
     name: Google Partner Monitoring
     category: Site Monitor
-    url:
+    url: ""
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Search Console) Chrome/41.0.2272.118 Safari/537.36
+  bot:
+    name: Google Search Console
+    category: Crawler
+    url: https://search.google.com/search-console/about
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)
+  bot:
+    name: Google Structured Data Testing Tool
+    category: Validator
+    url: https://search.google.com/structured-data/testing-tool
     producer:
       name: Google Inc.
       url: http://www.google.com
@@ -722,6 +1133,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
   bot:
@@ -758,6 +1178,33 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Google-AdWords-Express
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Google-Adwords-Instant (+http://www.google.com/adsbot.html)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Google-speakr
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Googlebot (gocrawl v0.4)
   bot:
@@ -839,6 +1286,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Google-Youtube-Links)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
   bot:
@@ -848,6 +1304,24 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (en-US) AppleWebKit/537.36 (KHTML, like Gecko; Google-Assess) Chrome/34.0.1847.116 Safari/537.36
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (en-us) AppleWebKit/537.36(KHTML, like Gecko; Google-Adwords-DisplayAds-WebRender;) Chrome/41.0.2272.118Safari/537.36
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
   bot:
@@ -866,6 +1340,33 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Google-AMPHTML)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; DuplexWeb-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)
   bot:
@@ -875,6 +1376,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Safari/537.36 (compatible; Google-HotelAdsVerifier/2.0)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36
   bot:
@@ -884,6 +1394,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Shopping-Quality +http://www.google.com/merchants/tos/extend/DE/tos.html)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36
   bot:
@@ -902,6 +1421,24 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +https://www.grapeshot.com/crawler/)
+  bot:
+    name: Grapeshot
+    category: Crawler
+    url: https://www.grapeshot.com/crawler
+    producer:
+      name: Grapeshot
+      url: https://www.grapeshot.com
+- 
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; GrapeshotCrawler/2.0; +https://www.grapeshot.com/crawler/)
+  bot:
+    name: Grapeshot
+    category: Crawler
+    url: https://www.grapeshot.com/crawler
+    producer:
+      name: Grapeshot
+      url: https://www.grapeshot.com
 - 
   user_agent: HTTPMon/1.0b (http://www.httpmon.com)
   bot:
@@ -983,6 +1520,15 @@
     producer:
       name: The Internet Archive
       url: http://www.archive.org
+- 
+  user_agent: Heurekabot-Feed/1.0 (+https://sluzby.heureka.cz/napoveda/heurekabot/)
+  bot:
+    name: Heureka Feed
+    category: Crawler
+    url: https://sluzby.heureka.cz/napoveda/heurekabot/
+    producer:
+      name: Heureka.cz, a.s.
+      url: https://www.heureka.cz/
 - 
   user_agent: HubPages V0.2.2 (http://hubpages.com/help/crawlingpolicy)
   bot:
@@ -992,6 +1538,14 @@
     producer:
       name: HubPages
       url: http://hubpages.com/
+- 
+  user_agent: HubSpot Website Grader (web-crawlers@hubspot.com)
+  bot:
+    name: HubSpot
+    category: Crawler
+    producer:
+      name: HubSpot Inc.
+      url: https://www.hubspot.com
 - 
   user_agent: ICC-Crawler/2.0 (Mozilla-compatible; ; http://www.nict.go.jp/en/univ-com/plan/crawl.html)
   bot:
@@ -999,8 +1553,17 @@
     category: Crawler
     url: http://www.nict.go.jp/en/univ-com/plan/crawl.html
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
+- 
+  user_agent: IDG/IT (http://spaziodati.eu/)
+  bot:
+    name: IDG/IT
+    category: Search bot
+    url: https://spaziodati.eu/
+    producer:
+      name: SpazioDati S.r.l.
+      url: https://spaziodati.eu/
 - 
   user_agent: iisbot/1.0 (+http://www.iis.net/iisbot.html)
   bot:
@@ -1010,6 +1573,32 @@
     producer:
       name: Microsoft Corporation
       url: http://www.microsoft.com
+- 
+  user_agent: IP-Guide.com Crawler/1.0 (https://ip-guide.com)
+  bot:
+    name: IP-Guide Crawler
+    category: Crawler
+    url: ""
+    producer:
+      name: ""
+      url: https://ip-guide.com
+- 
+  user_agent: Mozilla/5.0 (compatible; ips-agent)
+  bot:
+    name: IPS Agent
+    category: crawler
+    producer:
+      name: VeriSign, Inc
+      url: http://www.verisign.com/
+- 
+  user_agent: Kaspersky Lab CFR link resolver cfradmins@kaspersky.com
+  bot:
+    name: Kaspersky
+    category: Security Checker
+    url: https://www.kaspersky.com/
+    producer:
+      name: AO Kaspersky Lab
+      url: https://www.kaspersky.com/
 - 
   user_agent: kouio.com RSS reader
   bot:
@@ -1017,8 +1606,8 @@
     url: http://kouio.com/
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: 'kouio.com RSS reader - 6 subscribers'
   bot:
@@ -1026,16 +1615,25 @@
     url: http://kouio.com/
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
+- 
+  user_agent: LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)
+  bot:
+    name: LCC
+    category: Search bot
+    url: https://corpora.uni-leipzig.de/crawler_faq.html
+    producer:
+      name: Universität Leipzig
+      url: https://www.uni-leipzig.de/
 - 
   user_agent: 'ltx71 - (http://ltx71.com/)'
   bot:
     name: LTX71
     url: http://ltx71.com/
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: larbin_2.6.3 larbin2.6.3@unspecified.mail
   bot:
@@ -1043,8 +1641,26 @@
     category: Crawler
     url: http://larbin.sourceforge.net
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
+- 
+  user_agent: "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)"
+  bot:
+    name: "Let's Encrypt Validation"
+    category: Service Agent
+    url: https://letsencrypt.org/how-it-works/
+    producer:
+      name: "Let's Encrypt"
+      url: https://letsencrypt.org
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/61.0.3116.0 Mobile Safari/537.36 Chrome-Lighthouse
+  bot:
+    name: Lighthouse
+    category: Site Monitor
+    url: https://developers.google.com/web/tools/lighthouse
+    producer:
+      name: Lighthouse
+      url: https://developers.google.com/web/tools/lighthouse
 - 
   user_agent: linkdex.com/v2.0 and linkdex.com/v2.1
   bot:
@@ -1106,8 +1722,8 @@
     url: http://magpierss.sourceforge.net/
     category: Feed Parser
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
   bot:
@@ -1126,6 +1742,16 @@
     producer:
       name: Mail.Ru Group
       url: http://corp.mail.ru
+- 
+  user_agent: http.rb/2.2.2 (Mastodon/1.6.1; +https://mathtod.online/)
+  bot:
+    name: Mastodon Bot
+    category: Social Media Agent
+- 
+  user_agent: http.rb/3.2.0 (Mastodon/2.4.3; +https://uwu.social/)
+  bot:
+    name: Mastodon Bot
+    category: Social Media Agent
 - 
   user_agent: Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)
   bot:
@@ -1135,6 +1761,21 @@
     producer:
       name: Meanpath
       url: http://www.meanpath.com
+- 
+  user_agent: MetaInspector/5.4.0 (+https://github.com/jaimeiniesta/metainspector)
+  bot:
+    name: MetaInspector
+    category: Crawler
+    url: https://github.com/jaimeiniesta/metainspector
+- 
+  user_agent: Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)
+  bot:
+    name: MetaJobBot
+    category: Crawler
+    url: http://www.metajob.at/the/crawler
+    producer:
+      name: MetaJob
+      url: http://www.metajob.at
 - 
   user_agent: Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)
   bot:
@@ -1143,7 +1784,16 @@
     url: http://mixrank.com
     producer:
       name: Online Media Group, Inc.
-      url:
+      url: ""
+- 
+  user_agent: Mnogosearch-3.1.21
+  bot:
+    name: Mnogosearch
+    category: Search bot
+    url: http://www.mnogosearch.org/
+    producer:
+      name: Lavtech.Com Corp.
+      url: ""
 - 
   user_agent: Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)
   bot:
@@ -1162,15 +1812,42 @@
     producer:
       name: Monitor.Us
       url: http://www.monitor.us
+- 
+  user_agent: munin/2.0.30-1 (libwww-perl/6.15)
+  bot:
+    name: Munin
+    category: Site Monitor
+    url: http://munin-monitoring.org/
+    producer:
+      name: Munin
+      url: http://munin-monitoring.org/
+- 
+  user_agent: munin/http_loadtime
+  bot:
+    name: Munin
+    category: Site Monitor
+    url: http://munin-monitoring.org/
+    producer:
+      name: Munin
+      url: http://munin-monitoring.org/
 - 
   user_agent: nlcrawler/1.0 (+http://northernlight.com/)
   bot:
     name: NLCrawler
     category: Crawler
-    url:
+    url: ""
     producer:
       name: Northern Light
       url: http://northernlight.com
+- 
+  user_agent: check_http/v1.5 (nagios-plugins 1.5)
+  bot:
+    name: Nagios check_http
+    category: Site Monitor
+    url: https://nagios.org
+    producer:
+      name: Nagios Plugins Development Team
+      url: https://nagios.org
 - 
   user_agent: NalezenCzBot/1.0 (http://www.nalezen.cz/about-crawler)
   bot:
@@ -1179,7 +1856,7 @@
     url: http://www.nalezen.cz/about-crawler
     producer:
       name: Jaroslav Kuboš
-      url:
+      url: ""
 - 
   user_agent: NetLyzer FastProbe
   bot:
@@ -1189,7 +1866,25 @@
   bot:
     name: Netcraft Survey Bot
     category: Search bot
-    url:
+    url: ""
+    producer:
+      name: Netcraft
+      url: http://www.netcraft.com
+- 
+  user_agent: Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)
+  bot:
+    name: Netcraft Survey Bot
+    category: Search bot
+    url: ""
+    producer:
+      name: Netcraft
+      url: http://www.netcraft.com
+- 
+  user_agent: 'Netcraft SSL Server Survey - contact info@netcraft.com'
+  bot:
+    name: Netcraft Survey Bot
+    category: Search bot
+    url: ""
     producer:
       name: Netcraft
       url: http://www.netcraft.com
@@ -1200,8 +1895,8 @@
     url: http://www.netvibes.com/
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: 'Netvibes (http://www.netvibes.com/; 8 subscribers; feedID: 2244192)'
   bot:
@@ -1209,8 +1904,8 @@
     url: http://www.netvibes.com/
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: 'NewsBlur Favicon Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
   bot:
@@ -1218,8 +1913,8 @@
     url: http://www.newsblur.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: 'NewsBlur Feed Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
   bot:
@@ -1227,8 +1922,8 @@
     url: http://www.newsblur.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: NewsBlur Feed Finder (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
   bot:
@@ -1236,8 +1931,8 @@
     url: http://www.newsblur.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: 'NewsBlur Page Fetcher - 7 subscribers - http://www.newsblur.com/site/3966817/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
   bot:
@@ -1245,8 +1940,8 @@
     url: http://www.newsblur.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: NewsGatorOnline/2.0 (http://www.newsgator.com; 2 subscribers)
   bot:
@@ -1254,8 +1949,17 @@
     url: http://www.newsgator.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
+- 
+  user_agent: Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)
+  bot:
+    name: Nmap
+    category: Security Checker
+    url: https://nmap.org/book/nse.html
+    producer:
+      name: Nmap
+      url: https://nmap.org/
 - 
   user_agent: 5acd734a-1ed3-4a17-ad9d-0b09ae7bc2d2/Nutch-1.7
   bot:
@@ -1373,6 +2077,27 @@
     producer:
       name: The Apache Software Foundation
       url: http://www.apache.org/foundation/
+- 
+  user_agent: Nuzzel
+  bot:
+    name: Nuzzel
+    category: Crawler
+    producer:
+      name: Nuzzel
+      url: https://www.nuzzel.com/
+- 
+  user_agent: Octopus 1.0.2
+  bot:
+    name: Octopus
+- 
+  user_agent: omgili/0.5 +http://omgili.com
+  bot:
+    name: Omgili bot
+    category: Search bot
+    url: http://www.omgili.com/Crawler.html
+    producer:
+      name: Omgili
+      url: http://www.omgili.com
 - 
   user_agent: omgilibot/0.3 +http://www.omgili.com/Crawler.html
   bot:
@@ -1450,7 +2175,7 @@
   bot:
     name: Outbrain
     category: Crawler
-    url:
+    url: ""
     producer:
       name: Outbrain
       url: http://www.outbrain.com/
@@ -1472,6 +2197,12 @@
     producer:
       name: Smallrivers SA
       url: http://www.paper.li
+- 
+  user_agent: phantomas/1.11.0 (PhantomJS/1.9.8; linux x64)
+  bot:
+    name: Phantomas
+    category: Site Monitor
+    url: https://github.com/macbre/phantomas
 - 
   user_agent: psbot-page (+http://www.picsearch.com/bot.html)
   bot:
@@ -1495,19 +2226,46 @@
   bot:
     name: Pingdom Bot
     category: Site Monitor
-    url:
+    url: ""
     producer:
       name: Pingdom AB
       url: https://www.pingdom.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Pinterestbot/1.0; http://www.pinterest.com/bot.html)
+  bot:
+    name: Pinterest
+    url: http://www.pinterest.com/bot.html
+    category: Crawler
+    producer:
+      name: Pinterest
+      url: http://www.pinterest.com/
 - 
   user_agent: Pinterest/0.2 (+http://www.pinterest.com/)
   bot:
     name: Pinterest
-    url:
+    url: http://www.pinterest.com/bot.html
     category: Crawler
     producer:
       name: Pinterest
       url: http://www.pinterest.com/
+- 
+  user_agent: PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)
+  bot:
+    name: PocketParser
+    category: Read-it-later Service
+    url: https://getpocket.com/pocketparser_ua
+    producer:
+      name: Pocket
+      url: https://getpocket.com/
+- 
+  user_agent: PritTorrent/1.0
+  bot:
+    name: PritTorrent
+    category: Crawler
+    url: https://github.com/astro/prittorrent
+    producer:
+      name: Bitlove
+      url: http://bitlove.org/
 - 
   user_agent: QuerySeekerSpider ( http://queryseeker.com/bot.html )
   bot:
@@ -1517,15 +2275,52 @@
     producer:
       name: QueryEye Inc.
       url: http://queryeye.com
+- 
+  user_agent: Quora Link Preview/1.0 (http://www.quora.com)
+  bot:
+    name: Quora Link Preview
+    category: Crawler
+    url: ""
+    producer:
+      name: Quora
+      url: http://www.quora.com
+- 
+  user_agent: 'Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*'
+  bot:
+    name: Qwantify
+    category: Crawler
+    url: https://www.qwant.com/
+    producer:
+      name: Qwant Corporation
+      url: https://www.qwant.com/
 - 
   user_agent: ROI Hunter; https://api-dev.roihunter.com
   bot:
     name: ROI Hunter
     category: Crawler
-    url:
+    url: ""
     producer:
       name: Roihunter a.s.
       url: http://roihunter.com/
+- 
+  user_agent: RSSRadio (Push Notification Scanner;support@dorada.co.uk)
+  bot:
+    name: RSSRadio Bot
+- 
+  user_agent: Rainmeter WebParser plugin
+  bot:
+    name: Rainmeter
+    category: Crawler
+    url: https://www.rainmeter.net
+- 
+  user_agent: RamblerMail/6.0 (incompatible; ImageProxy/6.0)
+  bot:
+    name: RamblerMail Image Proxy
+    category: Crawler
+    url: ""
+    producer:
+      name: 'Rambler&Co'
+      url: https://rambler-co.ru/
 - 
   user_agent: Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)
   bot:
@@ -1571,6 +2366,15 @@
     producer:
       name: SEO Engine
       url: http://www.seoengine.com
+- 
+  user_agent: Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)
+  bot:
+    name: SEOkicks-Robot
+    category: Crawler
+    url: http://www.seokicks.de/robot.html
+    producer:
+      name: SEOkicks
+      url: https://www.seokicks.de/
 - 
   user_agent: Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)
   bot:
@@ -1580,6 +2384,51 @@
     producer:
       name: SISTRIX GmbH
       url: http://www.sistrix.de
+- 
+  user_agent: Mozilla/5.0 (compatible; SISTRIX Optimizer)
+  bot:
+    name: SISTRIX Optimizer
+    category: Crawler
+    url: https://optimizer.sistrix.com
+    producer:
+      name: SISTRIX GmbH
+      url: http://www.sistrix.de
+- 
+  user_agent: Mozilla/5.0 (compatible; SISTRIX Optimizer; Uptime; +https://www.sistrix.com/faq/uptime)
+  bot:
+    name: SISTRIX Optimizer
+    category: Crawler
+    url: https://optimizer.sistrix.com
+    producer:
+      name: SISTRIX GmbH
+      url: http://www.sistrix.de
+- 
+  user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
+  bot:
+    name: SMTBot
+    category: Search bot
+    url: https://www.similartech.com/smtbot
+    producer:
+      name: SimilarTech Ltd.
+      url: https://www.similartech.com/
+- 
+  user_agent: SSL Labs (https://www.ssllabs.com/about/assessment.html)
+  bot:
+    name: SSL Labs
+    category: Validator
+    url: https://www.ssllabs.com/about/assessment.html
+    producer:
+      name: SSL Labs
+      url: https://www.ssllabs.com/about/assessment.html
+- 
+  user_agent: SafeDNSBot (https://www.safedns.com/searchbot)
+  bot:
+    name: SafeDNSBot
+    category: Crawler
+    url: https://www.safedns.com/searchbot
+    producer:
+      name: SafeDNS, Inc.
+      url: https://www.safedns.com/
 - 
   user_agent: Scrapy/1.0.3.post6+g2d688cd (+http://scrapy.org)
   bot:
@@ -1602,8 +2451,8 @@
     category: Crawler
     url: http://www.screenerbot.com
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Mozilla/5.0 (compatible; SemrushBot/0.97; +http://www.semrush.com/bot.html)
   bot:
@@ -1617,11 +2466,24 @@
   user_agent: SensikaBot/x.33 (+http://sensika.com)
   bot:
     name: Sensika Bot
-    category:
-    url:
+    category: ""
+    url: ""
     producer:
       name: Sensika
       url: http://sensika.com
+- 
+  user_agent: sentry/8.6.0 (https://getsentry.com)
+  bot:
+    name: Sentry Bot
+    producer:
+      name: Sentry
+      url: https://sentry.io
+- 
+  user_agent: Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)
+  bot:
+    name: Seoscanners.net
+    category: Crawler
+    url: ""
 - 
   user_agent: Server Density Service Monitoring v2
   bot:
@@ -1662,6 +2524,33 @@
     producer:
       name: Seznam.cz, a.s.
       url: http://www.seznam.cz/
+- 
+  user_agent: Mozilla/5.0 SeznamEmailProxy/2.0.174
+  bot:
+    name: Seznam Email Proxy
+    category: Crawler
+    url: ""
+    producer:
+      name: Seznam.cz, a.s.
+      url: http://www.seznam.cz/
+- 
+  user_agent: Seznam-Zbozi-robot/3.0
+  bot:
+    name: Seznam Zbozi.cz
+    category: Crawler
+    url: ""
+    producer:
+      name: Seznam.cz, a.s.
+      url: https://www.zbozi.cz/
+- 
+  user_agent: Mozilla/5.0 (ShopAlike; LadenZeile) FeedBot
+  bot:
+    name: ShopAlike
+    category: Crawler
+    url: ""
+    producer:
+      name: Visual Meta
+      url: https://www.shopalike.cz/
 - 
   user_agent: ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)
   bot:
@@ -1671,6 +2560,15 @@
     producer:
       name: ShopWiki Corp.
       url: http://www.shopwiki.com
+- 
+  user_agent: shopify-partner-homepage-scraper
+  bot:
+    name: Shopify Partner
+    category: Crawler
+    url: https://www.shopify.com/partners
+    producer:
+      name: Shopify
+      url: https://www.shopify.com/
 - 
   user_agent: SilverReader/1.0; http://silverreader.com
   bot:
@@ -1678,17 +2576,8 @@
     url: http://silverreader.com
     category: Feed Fetcher
     producer:
-      name:
-      url:
-- 
-  user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
-  bot:
-    name: SimplePie
-    url: http://www.simplepie.org
-    category: Feed Parser
-    producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: SimplePie/1.2.1-dev (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20130514092120
   bot:
@@ -1696,8 +2585,8 @@
     url: http://www.simplepie.org
     category: Feed Parser
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20121030175911
   bot:
@@ -1705,8 +2594,8 @@
     url: http://www.simplepie.org
     category: Feed Parser
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Site24x7
   bot:
@@ -1716,12 +2605,45 @@
     producer:
       name: Site24x7
       url: https://www.site24x7.com
+- 
+  user_agent: SiteSucker for macOS/2.10.5
+  bot:
+    name: SiteSucker
+    category: Crawler
+    url: http://ricks-apps.com/osx/sitesucker/
+- 
+  user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com
+  bot:
+    name: Siteimprove
+    category: Search bot
+    url: https://siteimprove.com/
+    producer:
+      name: Siteimprove GmbH
+      url: https://siteimprove.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
+  bot:
+    name: Siteimprove
+    category: Search bot
+    url: https://siteimprove.com/
+    producer:
+      name: Siteimprove GmbH
+      url: https://siteimprove.com/
+- 
+  user_agent: sixy.ch/1.0
+  bot:
+    name: Sixy.ch
+    category: Site Monitor
+    url: http://sixy.ch
+    producer:
+      name: Manuel Kasper
+      url: https://neon1.net/
 - 
   user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5
   bot:
     name: Skype URI Preview
     category: Service Agent
-    url:
+    url: ""
     producer:
       name: Skype Communications S.à.r.l.
       url: https://www.skype.com
@@ -1734,6 +2656,15 @@
     producer:
       name: Slack Technologies
       url: http://slack.com
+- 
+  user_agent: 'AppEngine-Google; (+http://code.google.com/appengine; appid: s~snapchat-proxy)'
+  bot:
+    name: Snapchat Proxy
+    category: Crawler
+    url: https://www.snapchat.com
+    producer:
+      name: Snapchat Inc.
+      url: https://www.snapchat.com
 - 
   user_agent: New-Sogou-Spider/1.0 (compatible; MSIE 5.5; Windows 98)
   bot:
@@ -1779,6 +2710,12 @@
     producer:
       name: Tencent Holdings
       url: http://www.soso.com
+- 
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Sparkler/0.2.0-SNAPSHOT
+  bot:
+    name: Sparkler
+    category: Crawler
+    url: https://github.com/USCDataScience/sparkler
 - 
   user_agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.19; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/2010040121 Firefox/3.0.19
   bot:
@@ -1788,30 +2725,55 @@
     producer:
       name: Tailrank Inc
       url: http://spinn3r.com
+- 
+  user_agent: Spotify/1.0
+  bot:
+    name: Spotify
+    producer:
+      name: Spotify
+      url: https://www.spotify.com
 - 
   user_agent: Mozilla/5.0 (compatible; SputnikBot/2.2)
   bot:
     name: Sputnik Bot
-    category:
-    url:
+    category: ""
+    url: ""
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Mozilla/5.0 (compatible; SputnikImageBot/2.2)
   bot:
     name: Sputnik Bot
-    category:
-    url:
+    category: ""
+    url: ""
+    producer:
+      name: ""
+      url: ""
+- 
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) (compatible; Startpagina-Linkchecker/1.0; +https://www.startpagina.nl/linkchecker)
+  bot:
+    name: Startpagina Linkchecker
+    category: Search bot
+    url: https://www.startpagina.nl/linkchecker
+    producer:
+      name: Startpagina B.V.
+      url: https://www.startpagina.nl/
+- 
+  user_agent: Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake)
+  bot:
+    name: StatusCake
+    category: Site Monitor
+    url: https://www.statuscake.com
     producer:
-      name:
-      url:
+      name: StatusCake
+      url: https://www.statuscake.com
 - 
   user_agent: 'Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch!'
   bot:
     name: Superfeedr Bot
     category: Feed Fetcher
-    url:
+    url: ""
     producer:
       name: Superfeedr
       url: https://superfeedr.com/
@@ -1824,11 +2786,31 @@
     producer:
       name: Domain Tools
       url: http://www.domaintools.com
+- 
+  user_agent: TLSProbe/1.0 (+https://scan.trustnet.venafi.com/)
+  bot:
+    name: TLSProbe
+    url: https://scan.trustnet.venafi.com/
+    category: Security search bot
+    producer:
+      name: Venafi TrustNet
+      url: https://www.venafi.com
+- 
+  user_agent: Tarmot Gezgin/1.0 (compatible; TarmotGezgin/1.1; +http://www.tarmot.com/gezgin)
+  bot:
+    name: Tarmot Gezgin
+    url: http://www.tarmot.com/gezgin/
+    category: Search bot
 - 
   user_agent: TelegramBot (like TwitterBot)
   bot:
-    name: TelgramBot
+    name: TelegramBot
     url: https://telegram.org/blog/bot-revolution
+- 
+  user_agent: The Knowledge AI
+  bot:
+    name: The Knowledge AI
+    category: Crawler
 - 
   user_agent: TinEye-bot/0.02 (see http://www.tineye.com/crawler.html)
   bot:
@@ -1845,8 +2827,8 @@
     url: http://tt-rss.org
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
 - 
   user_agent: Tiny Tiny RSS/1.11.4c63934 (http://tt-rss.org/)
   bot:
@@ -1854,8 +2836,26 @@
     url: http://tt-rss.org
     category: Feed Fetcher
     producer:
-      name:
-      url:
+      name: ""
+      url: ""
+- 
+  user_agent: Mozilla/5.0 (compatible; tracemyfile/1.0)
+  bot:
+    name: TraceMyFile
+    category: Search bot
+    url: https://www.tracemyfile.com/
+    producer:
+      name: Idee Inc.
+      url: http://ideeinc.com/
+- 
+  user_agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11
+  bot:
+    name: Trendiction Bot
+    category: Crawler
+    url: http://www.trendiction.de/bot
+    producer:
+      name: Talkwalker Inc.
+      url: http://www.talkwalker.com
 - 
   user_agent: TurnitinBot/3.0 (http://www.turnitin.com/robot/crawlerinfo.html)
   bot:
@@ -1891,7 +2891,15 @@
     url: http://tweetmeme.com/
     producer:
       name: Mediasift
-      url:
+      url: ""
+- 
+  user_agent: Mozilla/5.0 (compatible; Twingly Recon; twingly.com)
+  bot:
+    name: Twingly Recon
+    category: Crawler
+    producer:
+      name: Twingly
+      url: https://www.twingly.com
 - 
   user_agent: Twitterbot/1.0
   bot:
@@ -1910,15 +2918,60 @@
     producer:
       name: Profound Networks
       url: http://www.profound.net
+- 
+  user_agent: 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 (via secureurl.fwdcdn.com - mail.ukr.net proxy)'
+  bot:
+    name: UkrNet Mail Proxy
+    category: Crawler
+    url: ""
+    producer:
+      name: UkrNet Ltd
+      url: https://www.ukr.net/
+- 
+  user_agent: UniversalFeedParser/5.2.1 +https://code.google.com/p/feedparser/
+  bot:
+    name: UniversalFeedParser
+    category: Feed Fetcher
+    url: https://github.com/kurtmckee/feedparser
+    producer:
+      name: Kurt McKee
+      url: https://github.com/kurtmckee
 - 
   user_agent: Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)
   bot:
     name: Uptime Robot
     category: Site Monitor
-    url:
+    url: ""
     producer:
       name: Uptime Robot
       url: http://uptimerobot.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)
+  bot:
+    name: Uptimebot
+    category: Site Monitor
+    url: https://uptime.com/uptimebot
+    producer:
+      name: Uptime
+      url: https://uptime.com
+- 
+  user_agent: Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)
+  bot:
+    name: VK Share Button
+    category: Crawler
+    url: http://vk.com/dev/Share
+    producer:
+      name: VK
+      url: http://vk.com/
+- 
+  user_agent: 'Mozilla/4.0 (compatible;  Vagabondo/4.0; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)'
+  bot:
+    name: Vagabondo
+    category: Crawler
+    url: ""
+    producer:
+      name: WiseGuys
+      url: http://www.wise-guys.nl/
 - 
   user_agent: Mozilla/5.0 (compatible; VSMCrawler; http://www.visualsitemapper.com/crawler/)
   bot:
@@ -1964,6 +3017,15 @@
     producer:
       name: W3C
       url: http://www.w3.org
+- 
+  user_agent: Validator.nu/LV http://validator.w3.org/services
+  bot:
+    name: W3C Markup Validation Service
+    category: Validator
+    url: http://validator.w3.org/services
+    producer:
+      name: W3C
+      url: http://www.w3.org
 - 
   user_agent: W3C_Validator/1.3 http://validator.w3.org/services
   bot:
@@ -2000,6 +3062,14 @@
     producer:
       name: W3C
       url: http://www.w3.org
+- 
+  user_agent: Mozilla/5.0 (compatible; Wappalyzer; +https://github.com/AliasIO/Wappalyzer)
+  bot:
+    name: Wappalyzer
+    url: https://github.com/AliasIO/Wappalyzer
+    producer:
+      name: AliasIO
+      url: https://github.com/AliasIO
 - 
   user_agent: WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)
   bot:
@@ -2009,6 +3079,12 @@
     producer:
       name: WeSEE Ltd
       url: http://www.wesee.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4) Build/MPJ24.139-64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Mobile Safari/537.36 PTST/180829.190838
+  bot:
+    name: WebPageTest
+    category: Site Monitor
+    url: https://www.webpagetest.org
 - 
   user_agent: websitepulse checker/1.1 (compatible; MSIE 5.5; Netscape 4.75; Linux)
   bot:
@@ -2027,6 +3103,51 @@
     producer:
       name: Steve Webb
       url: http://badcheese.com
+- 
+  user_agent: weborama-fetcher (+http://www.weborama.com)
+  bot:
+    name: Weborama
+    category: Search bot
+    url: https://weborama.com/
+    producer:
+      name: Weborama SA
+      url: https://weborama.com/
+- 
+  user_agent: WikiDo/1.1 (http://wikido.com; crawler@wikido.com)
+  bot:
+    name: WikiDo
+    category: Search bot
+    url: https://www.wikido.com/
+    producer:
+      name: Fotolitografie Fiorentine di Becchi Antonio s.n.c.
+      url: https://www.wikido.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)
+  bot:
+    name: WooRank
+    category: Search bot
+    url: https://www.woorank.com/
+    producer:
+      name: WooRank sprl
+      url: https://www.woorank.com/
+- 
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)
+  bot:
+    name: WooRank
+    category: Search bot
+    url: https://www.woorank.com/
+    producer:
+      name: WooRank sprl
+      url: https://www.woorank.com/
+- 
+  user_agent: WordPress/4.7.2; https://example.com
+  bot:
+    name: WordPress
+    category: Service Agent
+    url: https://wordpress.org/
+    producer:
+      name: Wordpress.org
+      url: https://wordpress.org/
 - 
   user_agent: Wotbox/2.01 (+http://www.wotbox.com/bot/)
   bot:
@@ -2036,6 +3157,15 @@
     producer:
       name: Wotbox
       url: http://www.wotbox.com
+- 
+  user_agent: XenForo/2.x (https://www.example.com)
+  bot:
+    name: XenForo
+    category: Service Agent
+    url: https://xenforo.com/
+    producer:
+      name: XenForo Ltd.
+      url: https://xenforo.com/
 - 
   user_agent: yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
   bot:
@@ -2063,6 +3193,15 @@
     producer:
       name: Yahoo! Inc.
       url: http://www.yahoo.com
+- 
+  user_agent: Y!J-BRW/1.0 (https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716)
+  bot:
+    name: Yahoo! Japan BRW
+    category: Crawler
+    url: https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/ウェブページにアクセスするシステムのユーザーエージェントについて
+    producer:
+      name: Yahoo! Japan Corp.
+      url: https://www.yahoo.co.jp/
 - 
   user_agent: Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)
   bot:
@@ -2207,6 +3346,15 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
 - 
   user_agent: Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)
   bot:
@@ -2225,6 +3373,87 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexVerticals/1.0; http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; NaverJapan/1.0; +http://corp.naver.jp/)
+  bot:
+    name: Yeti/Naverbot
+    category: Search bot
+    url: http://help.naver.com/robots/
+    producer:
+      name: Naver
+      url: http://www.naver.com
 - 
   user_agent: Yeti/1.1 (Naver Corp.; http://help.naver.com/robots/)
   bot:
@@ -2281,6 +3510,15 @@
   bot:
     name: Zao
     category: Crawler
+- 
+  user_agent: zelist.ro feed parser (+http://www.zelist.ro)
+  bot:
+    name: Ze List
+    category: Feed Fetcher
+    url: https://www.zelist.ro/
+    producer:
+      name: Treeworks SRL
+      url: https://www.tree.ro/
 - 
   user_agent: Zookabot/2.5;++http://zookabot.com
   bot:
@@ -2416,3 +3654,73 @@
     producer:
       name: reddit inc.
       url: http://www.reddit.com
+- 
+  user_agent: Mozilla/5.0 eCairn-Grabber/1.0 (+http://ecairn.com/grabber)
+  bot:
+    name: eCairn-Grabber
+    category: Crawler
+    producer:
+      name: eCairn
+      url: https://ecairn.com
+- 
+  user_agent: eZ Publish Link Validator
+  bot:
+    name: eZ Publish Link Validator
+    category: Crawler
+    url: https://ez.no/
+    producer:
+      name: eZ Systems AS
+      url: https://ez.no/
+- 
+  user_agent: Mozilla/5.0 (compatible; evc-batch/2.0)
+  bot:
+    name: evc-batch
+    category: Crawler
+    url: ""
+    producer:
+      name: eVenture Capital Partners II, LLC
+      url: http://www.eventures.vc/
+- 
+  user_agent: Mozilla/5.0 (compatible; inoreader.com; 2 subscribers)
+  bot:
+    name: inoreader
+    category: Feed Reader
+    url: https://www.inoreader.com
+- 
+  user_agent: masscan/1.0 (https://github.com/robertdavidgraham/masscan)
+  bot:
+    name: masscan
+    url: https://github.com/robertdavidgraham/masscan
+    category: Crawler
+    producer:
+      name: Robert Graham
+      url: https://github.com/robertdavidgraham
+- 
+  user_agent: 'Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com'
+  bot:
+    name: nbertaupete95
+    category: Crawler
+- 
+  user_agent: Mozilla/5.0 (compatible; oBot/2.3.1; http://www.xforce-security.com/crawler/)
+  bot:
+    name: oBot
+    category: Search bot
+    url: http://www.xforce-security.com/crawler/
+    producer:
+      name: 'IBM Germany Research & Development GmbH'
+      url: https://exchange.xforce.ibmcloud.com/
+- 
+  user_agent: 'sqlmap/1.1.8.2#dev (http://sqlmap.org)'
+  bot:
+    name: sqlmap
+    category: Security Checker
+    url: http://sqlmap.org/
+    producer:
+      name: sqlmap
+      url: http://sqlmap.org/
+- 
+  user_agent: Mozilla/5.0 (compatible; theoldreader.com; 1 subscribers; feed-id=aaa)
+  bot:
+    name: theoldreader
+    category: Feed Reader
+    url: https://theoldreader.com