device_detector 1.0.1 → 1.0.5

data/spec/fixtures/client/mobile_app.yml

@@ -1,110 +1,250 @@
--
+---
+- 
   user_agent: Pulse/4.0.5 (iPhone; iOS 7.0.6; Scale/2.00)
   client:
     type: mobile app
     name: Pulse
     version: "4.0.5"
--
+- 
   user_agent: Mozilla/5.0 (Linux; Android 4.4.4; SM-G7509 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 MicroMessenger/6.0.2.56_r958800.520 NetType/WIFI
   client:
     type: mobile app
     name: WeChat
     version: "6.0.2.56.r958800.520"
 -
+  user_agent: Mozilla/5.0 (iPad; CPU OS 11_2_6 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Mobile/15D100 AliApp(DingTalk/5.1.12) com.laiwang.DingTalk/13441885 Channel/201200 Pad/iPad language/zh-Hans-CN UT4Aplus/0.0.6 WK
+  client:
+    type: mobile app
+    name: DingTalk
+    version: 5.1.12
+- 
   user_agent: Mozilla/5.0 (Linux; U; Android 4.0.4; en-us; ZTE U807N Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 Weibo (ZTE-ZTE U807N__weibo__5.0.0__android__android4.0.4)
   client:
     type: mobile app
     name: Sina Weibo
     version: "5.0.0"
--
+- 
   user_agent: FeedR/1.0 (Android)
   client:
     type: mobile app
     name: FeedR
     version: "1.0"
--
+- 
   user_agent: Pinterest/3.3.3 CFNetwork/609.1.4 Darwin/13.0.0
   client:
     type: mobile app
     name: Pinterest
     version: "3.3.3"
--
+- 
   user_agent: Instacast/5.0a8 CFNetwork/711.1.11 Darwin/13.4.0
   client:
     type: mobile app
     name: Instacast
-    version: 5.0a8
--
+    version: "5.0a8"
+- 
   user_agent: InstacastHD/1.1.2 CFNetwork/711.0.6 Darwin/14.0.0
   client:
     type: mobile app
     name: Instacast
-    version: 1.1.2
--
+    version: "1.1.2"
+- 
   user_agent: Podcasts/2.1.2
   client:
     type: mobile app
     name: Podcasts
-    version: 2.1.2
--
+    version: "2.1.2"
+- 
   user_agent: Shifty Jelly Pocket Casts, Android v4.4.3.1
   client:
     type: mobile app
     name: Pocket Casts
-    version: 4.4.3.1
--
+    version: "4.4.3.1"
+- 
   user_agent: Pocket Casts
   client:
     type: mobile app
     name: Pocket Casts
-    version:
-
--
+    version: ""
+- 
   user_agent: Mozilla/5.0 (Linux; U; en-us; BeyondPod)
   client:
     type: mobile app
     name: BeyondPod
-    version:
--
+    version: ""
+- 
   user_agent: AntennaPod/0.9.9.1
   client:
     type: mobile app
     name: AntennaPod
-    version: 0.9.9.1
--
+    version: "0.9.9.1"
+- 
   user_agent: AntennaPod/
   client:
     type: mobile app
     name: AntennaPod
-    version: 
-
--
+    version: ""
+- 
   user_agent: Overcast/1.0 (+http://overcast.fm/; iOS podcast app)
   client:
     type: mobile app
     name: Overcast
-    version: 1.0
--
+    version: "1.0"
+- 
   user_agent: Podkicker/1.9.4
   client:
     type: mobile app
     name: Podkicker
-    version: 1.9.4
--
+    version: "1.9.4"
+- 
   user_agent: Podkicker Pro/1.9.4
   client:
     type: mobile app
     name: Podkicker
-    version: 1.9.4
--
+    version: "1.9.4"
+- 
   user_agent: Castro/64 CFNetwork/672.1.15 Darwin/14.0.0
   client:
     type: mobile app
     name: Castro
-    version: 64
--
+    version: "64"
+- 
   user_agent: Mozilla/5.0 (Linux; U; Windows NT 6.1; en-us; dream) DoggCatcher
   client:
     type: mobile app
     name: DoggCatcher
-    version:
+    version: ""
+- 
+  user_agent: Mozilla/5.0 (Linux; U; Android 4.2.2; ja-jp; SOL22 Build/10.3.1.D.0.257) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 YJApp-ANDROID jp.co.yahoo.android.yjtop/3.15.1
+  client:
+    type: mobile app
+    name: Yahoo! Japan
+    version: "3.15.1"
+- 
+  user_agent: RSSRadio/9220 (iPad;iOS;12.3.1)
+  client:
+    type: mobile app
+    name: RSSRadio
+    version: "9220"
+- 
+  user_agent: RSSRadio/9252
+  client:
+    type: mobile app
+    name: RSSRadio
+    version: "9252"
+- 
+  user_agent: RSSRadio/
+  client:
+    type: mobile app
+    name: RSSRadio
+    version: ""
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 7.0; JMM-AL00 Build/HONORJMM-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 SogouSearch Android1.0 version3.0 AppVersion/6002
+  client:
+    type: mobile app
+    name: SogouSearch App
+    version: "3.0"
+- 
+  user_agent: Dalvik/2.1.0 (Linux; U; Android 6.0.1; MI 4LTE MIUI/V7.3.2.0.MXDCNDD) NewsArticle/5.1.3
+  client:
+    type: mobile app
+    name: NewsArticle App
+    version: "5.1.3"
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 5.1; m2 note) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 baidubrowser/5.3.4.0 (Baidu; P1 4.3.1) tieba/6.7.2 BMW
+  client:
+    type: mobile app
+    name: tieba
+    version: "6.7.2"
+- 
+  user_agent: com.douban.group/2.2.0(220) Mozilla/5.0 (Linux; U; Android 4.1.1; zh-cn; MI 2A Build/JRO03L) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30
+  client:
+    type: mobile app
+    name: douban App
+    version: "2.2.0"
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 7.0; VIE-L09 Build/HUAWEIVIE-L09; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/59.0.3071.125 Mobile Safari/537.36 BingWeb/6.9.25207603
+  client:
+    type: mobile app
+    name: BingWebApp
+    version: "6.9.25207603"
+- 
+  user_agent: Mozilla/5.0 (Linux; U; Android 4.1.2; fr-ca; LT30a Build/9.1.A.0.489) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 GSA/3.3.11.1069658.arm
+  client:
+    type: mobile app
+    name: Google Search App
+    version: "3.3.11.1069658"
+- 
+  user_agent: Mozilla/5.0 (Linux; U; Android 2.3.4; ru-ru; WT19i Build/4.0.2.A.0.58) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 Flipboard/2.1.2/854,2.1.2.854,2013-12-05 11:59, +0300
+  client:
+    type: mobile app
+    name: Flipboard App
+    version: "2.1.2"
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; ASUS_Z011D Build/MMB29P; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36 Instagram 53.0.0.13.84 Android (23/6.0.1; 480dpi; 1080x1920; asus; ASUS_Z011D; ASUS_Z011; qcom; fr_FR; 116756948)
+  client:
+    type: mobile app
+    name: Instagram App
+    version: "53.0.0.13.84"
+- 
+  user_agent: TCL J706T_TD/1.0 Linux/3.4.5 Android/4.1.2 Release/08.27.2013 Browser/AppleWebKit534.30 Profile/MIDP-2.0 Configuration/CLDC-1.1 baiduboxapp/4.2 (Baidu; P1 4.1.2)
+  client:
+    type: mobile app
+    name: Baidu Box App
+    version: "4.2"
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 4.4.4; GT-I9195I Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Crosswalk/14.43.343.17 Mobile Safari/537.36
+  client:
+    type: mobile app
+    name: CrosswalkApp
+    version: "14.43.343.17"
+-
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.2.10 (KHTML, like Gecko) Mobile/15A372 Twitter for iPhone/7.52
+  client:
+    type: mobile app
+    name: Twitter
+    version: "7.52"
+-
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X; en-us) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone
+  client:
+    type: mobile app
+    name: Twitter
+    version: ""
+-
+  user_agent: Mozilla/5.0 (Linux; Android 5.1; HTC Desire 628 Build/LMY47D; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.108 Mobile Safari/537.36 JsSdk/2 TopBuzz/10.2.1 NetType/3G
+  client:
+    type: mobile app
+    name: TopBuzz
+    version: 10.2.1
+-
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Mobile/15E148 Snapchat/10.77.0.54 (like Safari/604.1)
+  client:
+    type: mobile app
+    name: Snapchat
+    version: 10.77.0.54
+
+-
+    user_agent: Siri/1 CFNetwork/1128.0.1 Darwin/19.6.0
+    client:
+      type: mobile app
+      name: Siri
+      version: "1.0"
+
+-
+  user_agent: LinkedIn/9.7.1681 CFNetwork/808.1.4 Darwin/16.1.0
+  client:
+    type: mobile app
+    name: LinkedIn
+    version: "9.7.1681"
+
+-
+  user_agent: Twitter/7.57 CFNetwork/978.0.7 Darwin/18.7.0
+  client:
+    type: mobile app
+    name: Twitter
+    version: "7.57"
+-
+  user_agent: Mozilla/5.0 (Linux; Android 9; SM-G950F Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/80.0.3987.117 Mobile Safari/537.36 TwitterAndroid
+  client:
+    type: mobile app
+    name: Twitter
+    version: ""

data/spec/fixtures/client/pim.yml

@@ -1,102 +1,115 @@
--
+---
+- 
   user_agent: Outlook-Express/7.0 (MSIE 7.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; AskTbORJ/5.15.9.29495; .NET4.0E; TmstmpExt)
   client:
     type: pim
     name: Outlook Express
     version: "7.0"
--
+- 
   user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Win64; x64; Trident/7.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3; .NET CLR 1.1.4322; FDM; Tablet PC 2.0; .NET4.0E; Microsoft Outlook 14.0.7113; ms-office; MSOffice 14)
   client:
     type: pim
     name: Microsoft Outlook
     version: "14.0.7113"
--
+- 
   user_agent: Mozilla/5.0 (X11; Linux i686; rv:17.0) Gecko/20130330 Thunderbird/17.0.5
   client:
     type: pim
     name: Thunderbird
     version: "17.0.5"
--
+- 
   user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Thunderbird/24.4.0 Lightning/2.6.4
   client:
     type: pim
     name: Thunderbird
     version: "24.4.0"
--
+- 
   user_agent: Airmail 1.4 rv:238 (Macintosh; Mac OS X 10.9.2; hr_HR)
   client:
     type: pim
     name: Airmail
     version: "1.4"
--
+- 
   user_agent: Airmail 1.3.3 rv:237 (Macintosh; Mac OS X 10.9.2; en_US)
   client:
     type: pim
     name: Airmail
     version: "1.3.3"
--
+- 
   user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Icedove/24.4.0
   client:
     type: pim
     name: Thunderbird
     version: "24.4.0"
--
+- 
   user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Icedove/24.4.0 Lightning/2.6.5
   client:
     type: pim
     name: Thunderbird
     version: "24.4.0"
--
+- 
   user_agent: Mozilla/4.0 (compatible; Lotus-Notes/6.0; Windows-NT)
   client:
     type: pim
     name: Lotus Notes
     version: "6.0"
--
+- 
   user_agent: Barca/2.8.4400
   client:
     type: pim
     name: Barca
     version: "2.8.4400"
--
+- 
   user_agent: BarcaPro/1.4 L.1001
   client:
     type: pim
     name: Barca
     version: "1.4"
--
+- 
   user_agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.8) Gecko/20100317 Postbox/1.1.3
   client:
     type: pim
     name: Postbox
     version: "1.1.3"
--
+- 
   user_agent: Postbox 1.0b14 (Windows/2009072715)
   client:
     type: pim
     name: Postbox
     version: "1.0"
--
+- 
   user_agent: MailBar/1.3.2 (Mac OS X Version 10.11.1 (Build 15B42))
   client:
     type: pim
     name: MailBar
-    version: 1.3.2
--
+    version: "1.3.2"
+- 
   user_agent: The Bat! 4.0.0.22
   client:
     type: pim
     name: The Bat!
     version: "4.0.0.22"
--
+- 
   user_agent: The Bat! Voyager 4.0.18.4
   client:
     type: pim
     name: The Bat!
     version: "4.0.18.4"
--
+- 
   user_agent: DAVdroid/1.6.2-ose (2017/06/23; dav4android; okhttp3) Android/7.0
   client:
     type: pim
     name: DAVdroid
-    version: "1.6.2"
+    version: "1.6.2"
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:10.0.12) Gecko/20130823 Firefox/10.0.11esrpre Iceape/2.7.12
+  client:
+    type: pim
+    name: SeaMonkey
+    version: "2.7.12"
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:29.0) Gecko/20100101 Firefox/29.0 SeaMonkey/2.26a1 Lightning/3.1a1
+  client:
+    type: pim
+    name: SeaMonkey
+    version: "2.26"

data/spec/fixtures/detector/bots.yml

@@ -75,6 +75,31 @@
     producer:
       name: Alexa Internet
       url: http://www.alexa.com
+- 
+  user_agent: Mozilla/5.0 (compatible; alexa site audit/1.0; http://www.alexa.com/help/webmasters; )
+  bot:
+    name: Alexa Site Audit
+    category: Site Monitor
+    url: http://www.alexa.com/help/webmasters
+    producer:
+      name: Alexa Internet
+      url: http://www.alexa.com
+- 
+  user_agent: Amazon Route 53 Health Check Service; ref:xxxxxxxx-xxxx-xxxxxxxxx-xxxxxxxxxxxx; report http://amzn.to/xxxxxxx
+  bot:
+    name: Amazon Route53 Health Check
+    category: Service Agent
+    producer:
+      name: Amazon Web Services
+      url: https://aws.amazon.com/
+- 
+  user_agent: Amazon-Route53-Health-Check-Service (ref b0eb04d5-cb5e-40e7-839b-558e52fc3f0d; report http://amzn.to/1vsZADi)
+  bot:
+    name: Amazon Route53 Health Check
+    category: Service Agent
+    producer:
+      name: Amazon Web Services
+      url: https://aws.amazon.com/
 - 
   user_agent: AmorankSpider/0.1; +http://amorank.com/webcrawler.html
   bot:
@@ -120,6 +145,15 @@
     producer:
       name: Apple Inc
       url: http://www.apple.com
+- 
+  user_agent: Arachni/v1.5.1
+  bot:
+    name: Arachni
+    category: Security Checker
+    url: http://www.arachni-scanner.com
+    producer:
+      name: Sarosys LLC
+      url: http://www.sarosys.com/
 - 
   user_agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
   bot:
@@ -129,6 +163,42 @@
     producer:
       name: Ask Jeeves Inc.
       url: http://www.ask.com
+- 
+  user_agent: Mozilla/5.0 (compatible;AspiegelBot)
+  bot:
+    name: AspiegelBot
+    category: Crawler
+    url: https://aspiegel.com/
+    producer:
+      name: Huawei
+      url: https://www.huawei.com/
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)
+  bot:
+    name: AspiegelBot
+    category: Crawler
+    url: https://aspiegel.com/
+    producer:
+      name: Huawei
+      url: https://www.huawei.com/
+- 
+  user_agent: AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)
+  bot:
+    name: Awario
+    category: Feed Fetcher
+    url: https://awario.com/bots.html
+    producer:
+      name: Awario
+      url: https://awario.com/
+- 
+  user_agent: AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)
+  bot:
+    name: Awario
+    category: Search bot
+    url: https://awario.com/bots.html
+    producer:
+      name: Awario
+      url: https://awario.com/
 - 
   user_agent: Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
   bot:
@@ -345,6 +415,23 @@
     producer:
       name: Blogtrottr Ltd
       url: https://blogtrottr.com/
+- 
+  user_agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com
+  bot:
+    name: BoardReader
+    category: Search bot
+    url: http://boardreader.com/
+    producer:
+      name: Effyis Inc
+      url: http://boardreader.com/
+- 
+  user_agent: BoardReader Blog Indexer(http://boardreader.com)
+  bot:
+    name: BoardReader Blog Indexer
+    category: Crawler
+    producer:
+      name: BoardReader
+      url: http://boardreader.com/
 - 
   user_agent: Mozilla/5.0 (compatible; BountiiBot/1.1; +http://bountii.com/contact.php)
   bot:
@@ -354,6 +441,15 @@
     producer:
       name: Bountii Inc.
       url: http://bountii.com
+- 
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)
+  bot:
+    name: BrandVerity
+    category: Crawler
+    url: https://www.brandverity.com/why-is-brandverity-visiting-me
+    producer:
+      name: BrandVerity, Inc.
+      url: https://www.brandverity.com/
 - 
   user_agent: Browsershots
   bot:
@@ -363,6 +459,15 @@
     producer:
       name: Browsershots.org
       url: http://browsershots.org
+- 
+  user_agent: Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)
+  bot:
+    name: Buck
+    category: Search bot
+    url: https://hypefactors.com/
+    producer:
+      name: Hypefactors A/S
+      url: https://hypefactors.com/
 - 
   user_agent: Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
   bot:
@@ -381,6 +486,24 @@
     producer:
       name: Topsy Labs
       url: http://labs.topsy.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.5668.1446 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com
+  bot:
+    name: Bytespider
+    category: Search bot
+    url: https://bytedance.com/
+    producer:
+      name: ByteDance Ltd.
+      url: https://bytedance.com/
+- 
+  user_agent: CATExplorador/1.0beta (sistemes at domini dot cat; http://domini.cat/catexplorador.html)
+  bot:
+    name: CATExplorador
+    category: Search bot
+    url: https://fundacio.cat/ca/domini/
+    producer:
+      name: Fundació puntCAT
+      url: https://fundacio.cat/ca/domini/
 - 
   user_agent: Mozilla/5.0 (compatible; CareerBot/1.1; +http://www.career-x.de/bot.html)
   bot:
@@ -430,8 +553,8 @@
   user_agent: collectd/5.5.1
   bot:
     name: Collectd
-    category: Site Monitor
     url: https://collectd.org/
+    category: Site Monitor
     producer:
       name: Collectd
       url: https://collectd.org/
@@ -444,15 +567,78 @@
     producer:
       name: ""
       url: ""
+- 
+  user_agent: coccocbot-web/1.0 (+http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
 - 
   user_agent: Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)
   bot:
     name: Cốc Cốc Bot
-    url: http://help.coccoc.com/
     category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-ads/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-fast/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-shopping/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
     producer:
       name: Cốc Cốc
-      url: http://coccoc.com/
+      url: https://coccoc.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; coccocbot/1.0; +http://help.coccoc.com/searchengine)
+  bot:
+    name: Cốc Cốc Bot
+    category: Search bot
+    url: https://help.coccoc.com/en/search-engine/coccoc-robots
+    producer:
+      name: Cốc Cốc
+      url: https://coccoc.com/
 - 
   user_agent: Datadog Agent/5.10.1
   bot:
@@ -462,6 +648,15 @@
     producer:
       name: Datadog
       url: https://www.datadoghq.com/
+- 
+  user_agent: Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36
+  bot:
+    name: Datanyze
+    url: ""
+    category: Crawler
+    producer:
+      name: Datanyze
+      url: https://www.datanyze.com
 - 
   user_agent: Mozilla/5.0 (compatible; Dataprovider/6.92; +https://www.dataprovider.com/)
   bot:
@@ -534,6 +729,15 @@
     producer:
       name: SEOmoz, Inc.
       url: http://moz.com/
+- 
+  user_agent: DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)
+  bot:
+    name: DuckDuckGo Bot
+    category: Search bot
+    url: https://duckduckgo.com/duckduckbot
+    producer:
+      name: DuckDuckGo
+      url: https://duckduckgo.com/
 - 
   user_agent: Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)
   bot:
@@ -570,6 +774,15 @@
     producer:
       name: easou ICP
       url: http://www.easou.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)
+  bot:
+    name: Embedly
+    category: Crawler
+    url: https://support.embed.ly/hc/en-us
+    producer:
+      name: A Medium, Corp.
+      url: https://medium.com/
 - 
   user_agent: Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)
   bot:
@@ -624,6 +837,15 @@
     producer:
       name: Facebook
       url: http://www.facebook.com
+- 
+  user_agent: facebookexternalua
+  bot:
+    name: Facebook External Hit
+    category: Social Media Agent
+    url: https://www.facebook.com/externalhit_uatext.php
+    producer:
+      name: Facebook
+      url: http://www.facebook.com
 - 
   user_agent: facebookplatform/1.0 (+http://developers.facebook.com)
   bot:
@@ -738,18 +960,59 @@
     producer:
       name: Flipboard
       url: http://flipboard.com/
+- 
+  user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
+  bot:
+    name: FreshRSS
+    category: Feed Fetcher
+    url: https://freshrss.org/
+- 
+  user_agent: FreshRSS/1.12.0 (Linux; https://freshrss.org)
+  bot:
+    name: FreshRSS
+    category: Feed Fetcher
+    url: https://freshrss.org/
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64; GTmetrix https://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36
+  bot:
+    name: GTmetrix
+    category: Crawler
+    url: https://gtmetrix.com/
+    producer:
+      name: Carbon60 Operating Co. Ltd.
+      url: https://www.carbon60.com/
+- 
+  user_agent: 7Siters/1.07 (+https://7ooo.ru/siters/)
+  bot:
+    name: Generic Bot
 - 
   user_agent: COMODO SSL Checker
   bot:
     name: Generic Bot
+- 
+  user_agent: help@dataminr.com
+  bot:
+    name: Generic Bot
 - 
   user_agent: 'HybridBot (hybrid.ru/about. If our bot caused problems please contact us. Contact email: m.lyashkov@targetix.net)'
   bot:
     name: Generic Bot
+- 
+  user_agent: 'Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)'
+  bot:
+    name: Generic Bot
+- 
+  user_agent: Mozilla/5.0 (compatible; +http://tweetedtimes.com)
+  bot:
+    name: Generic Bot
 - 
   user_agent: Mozilla/5.0 (compatible; Shareaholicbot/2.0; +http://www.shareaholic.com/bot)
   bot:
     name: Generic Bot
+- 
+  user_agent: Mozilla/5.0 (compatible; TrendsmapResolver/0.1)
+  bot:
+    name: Generic Bot
 - 
   user_agent: niki-bot
   bot:
@@ -758,6 +1021,14 @@
   user_agent: PHPCrawl
   bot:
     name: Generic Bot
+- 
+  user_agent: robots
+  bot:
+    name: Generic Bot
+- 
+  user_agent: SeopultContentAnalyzer/1.0
+  bot:
+    name: Generic Bot
 - 
   user_agent: Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)
   bot:
@@ -803,6 +1074,11 @@
     producer:
       name: NTT Resonant
       url: http://goo.ne.jp
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon
+  bot:
+    name: Google Favicon
+    category: Crawler
 - 
   user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0_1 like Mac OS X; en-us) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Version/4.0.5 Mobile/8A306 Safari/6531.22.7
   bot:
@@ -821,6 +1097,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Search Console) Chrome/41.0.2272.118 Safari/537.36
+  bot:
+    name: Google Search Console
+    category: Crawler
+    url: https://search.google.com/search-console/about
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)
   bot:
@@ -848,6 +1133,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
   bot:
@@ -884,6 +1178,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Google-AdWords-Express
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Google-Adwords-Instant (+http://www.google.com/adsbot.html)
   bot:
@@ -893,6 +1196,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Google-speakr
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Googlebot (gocrawl v0.4)
   bot:
@@ -974,6 +1286,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Google-Youtube-Links)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
   bot:
@@ -983,6 +1304,24 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (en-US) AppleWebKit/537.36 (KHTML, like Gecko; Google-Assess) Chrome/34.0.1847.116 Safari/537.36
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (en-us) AppleWebKit/537.36(KHTML, like Gecko; Google-Adwords-DisplayAds-WebRender;) Chrome/41.0.2272.118Safari/537.36
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
   bot:
@@ -1001,6 +1340,33 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Google-AMPHTML)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; DuplexWeb-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)
   bot:
@@ -1010,6 +1376,15 @@
     producer:
       name: Google Inc.
       url: http://www.google.com
+- 
+  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Safari/537.36 (compatible; Google-HotelAdsVerifier/2.0)
+  bot:
+    name: Googlebot
+    category: Search bot
+    url: http://www.google.com/bot.html
+    producer:
+      name: Google Inc.
+      url: http://www.google.com
 - 
   user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36
   bot:
@@ -1180,6 +1555,15 @@
     producer:
       name: ""
       url: ""
+- 
+  user_agent: IDG/IT (http://spaziodati.eu/)
+  bot:
+    name: IDG/IT
+    category: Search bot
+    url: https://spaziodati.eu/
+    producer:
+      name: SpazioDati S.r.l.
+      url: https://spaziodati.eu/
 - 
   user_agent: iisbot/1.0 (+http://www.iis.net/iisbot.html)
   bot:
@@ -1206,6 +1590,15 @@
     producer:
       name: VeriSign, Inc
       url: http://www.verisign.com/
+- 
+  user_agent: Kaspersky Lab CFR link resolver cfradmins@kaspersky.com
+  bot:
+    name: Kaspersky
+    category: Security Checker
+    url: https://www.kaspersky.com/
+    producer:
+      name: AO Kaspersky Lab
+      url: https://www.kaspersky.com/
 - 
   user_agent: kouio.com RSS reader
   bot:
@@ -1224,6 +1617,15 @@
     producer:
       name: ""
       url: ""
+- 
+  user_agent: LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)
+  bot:
+    name: LCC
+    category: Search bot
+    url: https://corpora.uni-leipzig.de/crawler_faq.html
+    producer:
+      name: Universität Leipzig
+      url: https://www.uni-leipzig.de/
 - 
   user_agent: 'ltx71 - (http://ltx71.com/)'
   bot:
@@ -1340,6 +1742,16 @@
     producer:
       name: Mail.Ru Group
       url: http://corp.mail.ru
+- 
+  user_agent: http.rb/2.2.2 (Mastodon/1.6.1; +https://mathtod.online/)
+  bot:
+    name: Mastodon Bot
+    category: Social Media Agent
+- 
+  user_agent: http.rb/3.2.0 (Mastodon/2.4.3; +https://uwu.social/)
+  bot:
+    name: Mastodon Bot
+    category: Social Media Agent
 - 
   user_agent: Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)
   bot:
@@ -1665,10 +2077,27 @@
     producer:
       name: The Apache Software Foundation
       url: http://www.apache.org/foundation/
+- 
+  user_agent: Nuzzel
+  bot:
+    name: Nuzzel
+    category: Crawler
+    producer:
+      name: Nuzzel
+      url: https://www.nuzzel.com/
 - 
   user_agent: Octopus 1.0.2
   bot:
     name: Octopus
+- 
+  user_agent: omgili/0.5 +http://omgili.com
+  bot:
+    name: Omgili bot
+    category: Search bot
+    url: http://www.omgili.com/Crawler.html
+    producer:
+      name: Omgili
+      url: http://www.omgili.com
 - 
   user_agent: omgilibot/0.3 +http://www.omgili.com/Crawler.html
   bot:
@@ -1801,11 +2230,20 @@
     producer:
       name: Pingdom AB
       url: https://www.pingdom.com
+- 
+  user_agent: Mozilla/5.0 (compatible; Pinterestbot/1.0; http://www.pinterest.com/bot.html)
+  bot:
+    name: Pinterest
+    url: http://www.pinterest.com/bot.html
+    category: Crawler
+    producer:
+      name: Pinterest
+      url: http://www.pinterest.com/
 - 
   user_agent: Pinterest/0.2 (+http://www.pinterest.com/)
   bot:
     name: Pinterest
-    url: ""
+    url: http://www.pinterest.com/bot.html
     category: Crawler
     producer:
       name: Pinterest
@@ -1837,6 +2275,15 @@
     producer:
       name: QueryEye Inc.
       url: http://queryeye.com
+- 
+  user_agent: Quora Link Preview/1.0 (http://www.quora.com)
+  bot:
+    name: Quora Link Preview
+    category: Crawler
+    url: ""
+    producer:
+      name: Quora
+      url: http://www.quora.com
 - 
   user_agent: 'Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*'
   bot:
@@ -1855,6 +2302,10 @@
     producer:
       name: Roihunter a.s.
       url: http://roihunter.com/
+- 
+  user_agent: RSSRadio (Push Notification Scanner;support@dorada.co.uk)
+  bot:
+    name: RSSRadio Bot
 - 
   user_agent: Rainmeter WebParser plugin
   bot:
@@ -1933,6 +2384,42 @@
     producer:
       name: SISTRIX GmbH
       url: http://www.sistrix.de
+-
+  user_agent: Mozilla/5.0 (compatible; Optimizer)
+  bot:
+    name: SISTRIX Optimizer
+    category: Crawler
+    url: https://optimizer.sistrix.com
+    producer:
+      name: SISTRIX GmbH
+      url: http://www.sistrix.de
+-
+  user_agent: Mozilla/5.0 (compatible; SISTRIX Optimizer)
+  bot:
+    name: SISTRIX Optimizer
+    category: Crawler
+    url: https://optimizer.sistrix.com
+    producer:
+      name: SISTRIX GmbH
+      url: http://www.sistrix.de
+- 
+  user_agent: Mozilla/5.0 (compatible; SISTRIX Optimizer; Uptime; +https://www.sistrix.com/faq/uptime)
+  bot:
+    name: SISTRIX Optimizer
+    category: Crawler
+    url: https://optimizer.sistrix.com
+    producer:
+      name: SISTRIX GmbH
+      url: http://www.sistrix.de
+- 
+  user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
+  bot:
+    name: SMTBot
+    category: Search bot
+    url: https://www.similartech.com/smtbot
+    producer:
+      name: SimilarTech Ltd.
+      url: https://www.similartech.com/
 - 
   user_agent: SSL Labs (https://www.ssllabs.com/about/assessment.html)
   bot:
@@ -2082,6 +2569,15 @@
     producer:
       name: ShopWiki Corp.
       url: http://www.shopwiki.com
+- 
+  user_agent: shopify-partner-homepage-scraper
+  bot:
+    name: Shopify Partner
+    category: Crawler
+    url: https://www.shopify.com/partners
+    producer:
+      name: Shopify
+      url: https://www.shopify.com/
 - 
   user_agent: SilverReader/1.0; http://silverreader.com
   bot:
@@ -2091,15 +2587,6 @@
     producer:
       name: ""
       url: ""
-- 
-  user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
-  bot:
-    name: SimplePie
-    url: http://www.simplepie.org
-    category: Feed Parser
-    producer:
-      name: ""
-      url: ""
 - 
   user_agent: SimplePie/1.2.1-dev (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20130514092120
   bot:
@@ -2133,6 +2620,24 @@
     name: SiteSucker
     category: Crawler
     url: http://ricks-apps.com/osx/sitesucker/
+- 
+  user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com
+  bot:
+    name: Siteimprove
+    category: Search bot
+    url: https://siteimprove.com/
+    producer:
+      name: Siteimprove GmbH
+      url: https://siteimprove.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
+  bot:
+    name: Siteimprove
+    category: Search bot
+    url: https://siteimprove.com/
+    producer:
+      name: Siteimprove GmbH
+      url: https://siteimprove.com/
 - 
   user_agent: sixy.ch/1.0
   bot:
@@ -2160,6 +2665,15 @@
     producer:
       name: Slack Technologies
       url: http://slack.com
+- 
+  user_agent: 'AppEngine-Google; (+http://code.google.com/appengine; appid: s~snapchat-proxy)'
+  bot:
+    name: Snapchat Proxy
+    category: Crawler
+    url: https://www.snapchat.com
+    producer:
+      name: Snapchat Inc.
+      url: https://www.snapchat.com
 - 
   user_agent: New-Sogou-Spider/1.0 (compatible; MSIE 5.5; Windows 98)
   bot:
@@ -2220,6 +2734,13 @@
     producer:
       name: Tailrank Inc
       url: http://spinn3r.com
+- 
+  user_agent: Spotify/1.0
+  bot:
+    name: Spotify
+    producer:
+      name: Spotify
+      url: https://www.spotify.com
 - 
   user_agent: Mozilla/5.0 (compatible; SputnikBot/2.2)
   bot:
@@ -2238,6 +2759,15 @@
     producer:
       name: ""
       url: ""
+- 
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) (compatible; Startpagina-Linkchecker/1.0; +https://www.startpagina.nl/linkchecker)
+  bot:
+    name: Startpagina Linkchecker
+    category: Search bot
+    url: https://www.startpagina.nl/linkchecker
+    producer:
+      name: Startpagina B.V.
+      url: https://www.startpagina.nl/
 - 
   user_agent: Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake)
   bot:
@@ -2283,8 +2813,13 @@
 - 
   user_agent: TelegramBot (like TwitterBot)
   bot:
-    name: TelgramBot
+    name: TelegramBot
     url: https://telegram.org/blog/bot-revolution
+- 
+  user_agent: The Knowledge AI
+  bot:
+    name: The Knowledge AI
+    category: Crawler
 - 
   user_agent: TinEye-bot/0.02 (see http://www.tineye.com/crawler.html)
   bot:
@@ -2312,6 +2847,15 @@
     producer:
       name: ""
       url: ""
+- 
+  user_agent: Mozilla/5.0 (compatible; tracemyfile/1.0)
+  bot:
+    name: TraceMyFile
+    category: Search bot
+    url: https://www.tracemyfile.com/
+    producer:
+      name: Idee Inc.
+      url: http://ideeinc.com/
 - 
   user_agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11
   bot:
@@ -2357,6 +2901,14 @@
     producer:
       name: Mediasift
       url: ""
+- 
+  user_agent: Mozilla/5.0 (compatible; Twingly Recon; twingly.com)
+  bot:
+    name: Twingly Recon
+    category: Crawler
+    producer:
+      name: Twingly
+      url: https://www.twingly.com
 - 
   user_agent: Twitterbot/1.0
   bot:
@@ -2411,6 +2963,15 @@
     producer:
       name: Uptime
       url: https://uptime.com
+- 
+  user_agent: Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)
+  bot:
+    name: VK Share Button
+    category: Crawler
+    url: http://vk.com/dev/Share
+    producer:
+      name: VK
+      url: http://vk.com/
 - 
   user_agent: 'Mozilla/4.0 (compatible;  Vagabondo/4.0; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)'
   bot:
@@ -2527,6 +3088,12 @@
     producer:
       name: WeSEE Ltd
       url: http://www.wesee.com
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4) Build/MPJ24.139-64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Mobile Safari/537.36 PTST/180829.190838
+  bot:
+    name: WebPageTest
+    category: Site Monitor
+    url: https://www.webpagetest.org
 - 
   user_agent: websitepulse checker/1.1 (compatible; MSIE 5.5; Netscape 4.75; Linux)
   bot:
@@ -2545,6 +3112,42 @@
     producer:
       name: Steve Webb
       url: http://badcheese.com
+- 
+  user_agent: weborama-fetcher (+http://www.weborama.com)
+  bot:
+    name: Weborama
+    category: Search bot
+    url: https://weborama.com/
+    producer:
+      name: Weborama SA
+      url: https://weborama.com/
+- 
+  user_agent: WikiDo/1.1 (http://wikido.com; crawler@wikido.com)
+  bot:
+    name: WikiDo
+    category: Search bot
+    url: https://www.wikido.com/
+    producer:
+      name: Fotolitografie Fiorentine di Becchi Antonio s.n.c.
+      url: https://www.wikido.com/
+- 
+  user_agent: Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)
+  bot:
+    name: WooRank
+    category: Search bot
+    url: https://www.woorank.com/
+    producer:
+      name: WooRank sprl
+      url: https://www.woorank.com/
+- 
+  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)
+  bot:
+    name: WooRank
+    category: Search bot
+    url: https://www.woorank.com/
+    producer:
+      name: WooRank sprl
+      url: https://www.woorank.com/
 - 
   user_agent: WordPress/4.7.2; https://example.com
   bot:
@@ -2563,6 +3166,15 @@
     producer:
       name: Wotbox
       url: http://www.wotbox.com
+- 
+  user_agent: XenForo/2.x (https://www.example.com)
+  bot:
+    name: XenForo
+    category: Service Agent
+    url: https://xenforo.com/
+    producer:
+      name: XenForo Ltd.
+      url: https://xenforo.com/
 - 
   user_agent: yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
   bot:
@@ -2590,6 +3202,15 @@
     producer:
       name: Yahoo! Inc.
       url: http://www.yahoo.com
+- 
+  user_agent: Y!J-BRW/1.0 (https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716)
+  bot:
+    name: Yahoo! Japan BRW
+    category: Crawler
+    url: https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/ウェブページにアクセスするシステムのユーザーエージェントについて
+    producer:
+      name: Yahoo! Japan Corp.
+      url: https://www.yahoo.co.jp/
 - 
   user_agent: Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)
   bot:
@@ -2734,6 +3355,15 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
 - 
   user_agent: Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)
   bot:
@@ -2752,6 +3382,87 @@
     producer:
       name: Yandex LLC
       url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; YandexVerticals/1.0; http://yandex.com/bots)
+  bot:
+    name: Yandex Bot
+    category: Search bot
+    url: http://www.yandex.com/bots
+    producer:
+      name: Yandex LLC
+      url: http://company.yandex.com
+- 
+  user_agent: Mozilla/5.0 (compatible; NaverJapan/1.0; +http://corp.naver.jp/)
+  bot:
+    name: Yeti/Naverbot
+    category: Search bot
+    url: http://help.naver.com/robots/
+    producer:
+      name: Naver
+      url: http://www.naver.com
 - 
   user_agent: Yeti/1.1 (Naver Corp.; http://help.naver.com/robots/)
   bot:
@@ -2808,6 +3519,15 @@
   bot:
     name: Zao
     category: Crawler
+- 
+  user_agent: zelist.ro feed parser (+http://www.zelist.ro)
+  bot:
+    name: Ze List
+    category: Feed Fetcher
+    url: https://www.zelist.ro/
+    producer:
+      name: Treeworks SRL
+      url: https://www.tree.ro/
 - 
   user_agent: Zookabot/2.5;++http://zookabot.com
   bot:
@@ -2943,6 +3663,23 @@
     producer:
       name: reddit inc.
       url: http://www.reddit.com
+- 
+  user_agent: Mozilla/5.0 eCairn-Grabber/1.0 (+http://ecairn.com/grabber)
+  bot:
+    name: eCairn-Grabber
+    category: Crawler
+    producer:
+      name: eCairn
+      url: https://ecairn.com
+- 
+  user_agent: eZ Publish Link Validator
+  bot:
+    name: eZ Publish Link Validator
+    category: Crawler
+    url: https://ez.no/
+    producer:
+      name: eZ Systems AS
+      url: https://ez.no/
 - 
   user_agent: Mozilla/5.0 (compatible; evc-batch/2.0)
   bot:
@@ -2952,15 +3689,35 @@
     producer:
       name: eVenture Capital Partners II, LLC
       url: http://www.eventures.vc/
+- 
+  user_agent: Mozilla/5.0 (compatible; inoreader.com; 2 subscribers)
+  bot:
+    name: inoreader
+    category: Feed Reader
+    url: https://www.inoreader.com
 - 
   user_agent: masscan/1.0 (https://github.com/robertdavidgraham/masscan)
   bot:
     name: masscan
-    category: Crawler
     url: https://github.com/robertdavidgraham/masscan
+    category: Crawler
     producer:
       name: Robert Graham
       url: https://github.com/robertdavidgraham
+- 
+  user_agent: 'Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com'
+  bot:
+    name: nbertaupete95
+    category: Crawler
+- 
+  user_agent: Mozilla/5.0 (compatible; oBot/2.3.1; http://www.xforce-security.com/crawler/)
+  bot:
+    name: oBot
+    category: Search bot
+    url: http://www.xforce-security.com/crawler/
+    producer:
+      name: 'IBM Germany Research & Development GmbH'
+      url: https://exchange.xforce.ibmcloud.com/
 - 
   user_agent: 'sqlmap/1.1.8.2#dev (http://sqlmap.org)'
   bot:
@@ -2971,8 +3728,86 @@
       name: sqlmap
       url: http://sqlmap.org/
 - 
-  user_agent: Mozilla/5.0 zgrab/0.x
+  user_agent: Mozilla/5.0 (compatible; theoldreader.com; 1 subscribers; feed-id=aaa)
   bot:
-    name: zgrab
-    category: Security Checker
-    url: https://github.com/zmap/zgrab
+    name: theoldreader
+    category: Feed Reader
+    url: https://theoldreader.com
+
+-
+  user_agent: Seobility
+  bot:
+    name: Seobility
+    category: Crawler
+    url: 'https://www.seobility.net/en/faq/?category=crawling#!aboutourbot'
+
+-
+  user_agent: Vercelbot (+https://vercel.com)
+  bot:
+    name: Vercel Bot
+    category: Service bot
+    url: https://vercel.com
+
+-
+  user_agent: Grammarly/1.0 (http://www.grammarly.com)
+  bot:
+    name: Grammarly
+    category: Service bot
+    url: http://www.grammarly.com
+
+-
+  user_agent: Robozilla/1.0
+  bot:
+    name: Robozilla
+    category: Crawler
+
+-
+  user_agent: Mozilla/5.0 (compatible; Domains Project/1.1.0; +https://domainsproject.org)
+  bot:
+    name: Domains Project
+    category: Crawler
+    url: https://domainsproject.org
+
+-
+  user_agent: Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://aspiegel.com/petalbot)
+  bot:
+    name: Petal Bot
+    category: Crawler
+    url: https://aspiegel.com/petalbot
+
+-
+  user_agent: SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)
+  bot:
+    name: Serendeputy Bot
+    category: Crawler
+    url: http://serendeputy.com/about/serendeputy-bot
+
+-
+  user_agent: ias-va/3.1 (+https://www.admantx.com/service-fetcher.html)
+  bot:
+    name: ADmantX Service Fetcher
+    category: Service bot
+    url: https://www.admantx.com/service-fetcher.html
+
+-
+  user_agent: Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)
+  bot:
+    name: Semantic Scholar Bot
+    category: Crawler
+    url: https://www.semanticscholar.org/crawler
+
+-
+  user_agent: Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)
+  bot:
+    name: Velen Public Web Crawler
+    category: Crawler
+    url: https://hunter.io/robot
+
+-
+  user_agent: Barkrowler/0.9 (+http://www.exensa.com/crawl)
+  bot:
+    name: Barkrowler
+    category: Crawler
+    url: http://www.exensa.com/crawl
+
+