device_detector 0.9.1 → 1.0.4

data/lib/device_detector/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class DeviceDetector
-  VERSION = '0.9.1'
+  VERSION = '1.0.4'
 end

data/lib/device_detector/version_extractor.rb

@@ -1,12 +1,11 @@
+# frozen_string_literal: true
+
 class DeviceDetector
   class VersionExtractor < MetadataExtractor
-
     private
 
     def metadata_string
       String(regex_meta[:version])
     end
-
   end
 end
-

data/regexes/bots.yml

@@ -1,7 +1,7 @@
 ###############
 # Device Detector - The Universal Device Detection library for parsing User Agents
 #
-# @link http://piwik.org
+# @link https://matomo.org
 # @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later
 ###############
 
@@ -53,6 +53,21 @@
     name: 'Alexa Internet'
     url: 'http://www.alexa.com'
 
+- regex: 'alexa site audit'
+  name: 'Alexa Site Audit'
+  category: 'Site Monitor'
+  url: 'http://www.alexa.com/help/webmasters'
+  producer:
+    name: 'Alexa Internet'
+    url: 'http://www.alexa.com'
+
+- regex: 'Amazon[ -]Route ?53[ -]Health[ -]Check[ -]Service'
+  name: 'Amazon Route53 Health Check'
+  category: 'Service Agent'
+  producer:
+    name: 'Amazon Web Services'
+    url: 'https://aws.amazon.com/'
+
 - regex: 'AmorankSpider'
   name: 'Amorank Spider'
   category: 'Crawler'
@@ -61,6 +76,14 @@
     name: 'Amorank'
     url: 'http://www.amorank.com'
 
+- regex: 'ApacheBench'
+  name: 'ApacheBench'
+  category: 'Benchmark'
+  url: 'https://httpd.apache.org/docs/2.4/programs/ab.html'
+  producer:
+    name: 'The Apache Software Foundation'
+    url: 'http://www.apache.org/foundation/'
+
 - regex: 'Applebot'
   name: 'Applebot'
   category: 'Crawler'
@@ -69,6 +92,30 @@
     name: 'Apple Inc'
     url: 'http://www.apple.com'
 
+- regex: 'Arachni'
+  name: 'Arachni'
+  category: 'Security Checker'
+  url: 'http://www.arachni-scanner.com'
+  producer:
+    name: 'Sarosys LLC'
+    url: 'http://www.sarosys.com/'
+
+- regex: 'AspiegelBot'
+  name: 'AspiegelBot'
+  category: 'Crawler'
+  url: 'https://aspiegel.com/'
+  producer:
+    name: 'Huawei'
+    url: 'https://www.huawei.com/'
+
+- regex: 'Castro 2, Episode Duration Lookup'
+  name: 'Castro 2'
+  category: 'Service Agent'
+  url: 'http://supertop.co/castro/'
+  producer: 
+    name: 'Supertop'
+    url: 'http://supertop.co'
+
 - regex: 'Curious George'
   name: 'Analytics SEO Crawler'
   category: 'Crawler'
@@ -93,8 +140,8 @@
     name: 'Ask Jeeves Inc.'
     url: 'http://www.ask.com'
 
-- regex: 'Backlink-Ceck\.de'
-  name: 'Backlink-Ceck.de'
+- regex: 'Backlink-Check\.de'
+  name: 'Backlink-Check.de'
   category: 'Crawler'
   url: 'http://www.backlink-check.de/bot.html'
   producer:
@@ -165,6 +212,13 @@
     name: 'Blogtrottr Ltd'
     url: 'https://blogtrottr.com/'
 
+- regex: 'BoardReader Blog Indexer'
+  name: 'BoardReader Blog Indexer'
+  category: 'Crawler'
+  producer:
+    name: 'BoardReader'
+    url: 'http://boardreader.com/'
+
 - regex: 'BountiiBot'
   name: 'Bountii Bot'
   category: 'Search bot'
@@ -186,8 +240,8 @@
   category: 'Crawler'
   url: 'http://law.di.unimi.it/BUbiNG.html'
   producer:
-    name: ''
-    url: ''
+    name: 'The Laboratory for Web Algorithmics (LAW)'
+    url: 'http://law.di.unimi.it/software.php#buging'
 
 - regex: '(?<!HTC)[ _]Butterfly/'
   name: 'Butterfly Robot'
@@ -221,6 +275,14 @@
     name: '10betterpages GmbH'
     url: 'http://cliqz.com'
 
+- regex: 'Cloudflare-AMP'
+  name: 'CloudFlare AMP Fetcher'
+  category: 'Crawler'
+  url: 'https://amp.cloudflare.com/doc/fetcher.html'
+  producer:
+    name: 'CloudFlare'
+    url: 'http://www.cloudflare.com'
+
 - regex: 'CloudFlare-AlwaysOnline'
   name: 'CloudFlare Always Online'
   category: 'Site Monitor'
@@ -229,13 +291,21 @@
     name: 'CloudFlare'
     url: 'http://www.cloudflare.com'
 
-- regex: 'coccoc/'
+- regex: 'coccoc|coccocbot(-ads|-fast|-image|-shopping|-web)?'
   name: 'Cốc Cốc Bot'
-  url: 'http://help.coccoc.com/'
+  url: 'https://help.coccoc.com/en/search-engine/coccoc-robots'
   category: 'Search bot'
   producer:
     name: 'Cốc Cốc'
-    url: 'http://coccoc.com/'
+    url: 'https://coccoc.com/'
+
+- regex: 'collectd'
+  name: 'Collectd'
+  url: 'https://collectd.org/'
+  category: 'Site Monitor'
+  producer:
+    name: 'Collectd'
+    url: 'https://collectd.org/'
 
 - regex: 'CommaFeed'
   name: 'CommaFeed'
@@ -245,6 +315,39 @@
     name: ''
     url: ''
 
+- regex: 'CSS Certificate Spider'
+  name: 'CSS Certificate Spider'
+  category: 'Crawler'
+  url: 'http://www.css-security.com/certificatespider/'
+  producer:
+    name: 'Certified Security Solutions'
+    url: 'https://www.css-security.com/company/about-us/'
+
+- regex: 'Datadog Agent'
+  name: 'Datadog Agent'
+  url: 'https://github.com/DataDog/dd-agent'
+  category: 'Site Monitor'
+  producer:
+    name: 'Datadog'
+    url: 'https://www.datadoghq.com/'
+
+- regex: 'Datanyze'
+  name: 'Datanyze'
+  url: ''
+  category: 'Crawler'
+  producer:
+    name: 'Datanyze'
+    url: 'https://www.datanyze.com'
+
+
+- regex: 'Dataprovider'
+  name: 'Dataprovider'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'Dataprovider B.V.'
+    url: 'https://www.dataprovider.com/'
+
 - regex: 'Daum(oa)?[ /][0-9]'
   name: 'Daum'
   category: 'Search bot'
@@ -285,6 +388,14 @@
     name: 'SEOmoz, Inc.'
     url: 'http://moz.com/'
 
+- regex: 'DuckDuck(?:Go-Favicons-)?Bot'
+  name: 'DuckDuckGo Bot'
+  category: 'Search bot'
+  url: 'https://duckduckgo.com/duckduckbot'
+  producer:
+    name: 'DuckDuckGo'
+    url: 'https://duckduckgo.com/'
+
 - regex: 'EasouSpider'
   name: 'Easou Spider'
   category: 'Search bot'
@@ -293,6 +404,13 @@
     name: 'easou ICP'
     url: 'http://www.easou.com'
 
+- regex: 'eCairn-Grabber'
+  name: 'eCairn-Grabber'
+  category: 'Crawler'
+  producer:
+    name: 'eCairn'
+    url: 'https://ecairn.com'
+
 - regex: 'EMail Exractor'
   name: 'EMail Exractor'
   category: 'Crawler'
@@ -301,6 +419,14 @@
     name: ''
     url: ''
 
+- regex: 'evc-batch'
+  name: 'evc-batch'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'eVenture Capital Partners II, LLC'
+    url: 'http://www.eventures.vc/'
+
 - regex: 'Exabot(-Thumbnails|-Images)?|ExaleadCloudview'
   name: 'ExaBot'
   category: 'Crawler'
@@ -325,7 +451,7 @@
     name: 'SEOmoz, Inc.'
     url: 'http://moz.com/'
 
-- regex: 'facebookexternalhit|facebookplatform'
+- regex: 'facebookexternalhit|facebookplatform|facebookexternalua'
   name: 'Facebook External Hit'
   category: 'Social Media Agent'
   url: 'https://www.facebook.com/externalhit_uatext.php'
@@ -381,6 +507,24 @@
     name: ''
     url: ''
 
+- regex: 'FlipboardProxy|FlipboardRSS'
+  name: 'Flipboard'
+  url: 'http://flipboard.com/browserproxy'
+  category: 'Feed Fetcher'
+  producer:
+    name: 'Flipboard'
+    url: 'http://flipboard.com/'
+
+- regex: 'Findxbot'
+  name: 'Findxbot'
+  category: 'Crawler'
+  url: 'http://www.findxbot.com'
+
+- regex: 'FreshRSS'
+  name: 'FreshRSS'
+  category: 'Feed Fetcher'
+  url: 'https://freshrss.org/'
+
 - regex: 'Genieo'
   name: 'Genieo Web filter'
   category: ''
@@ -389,6 +533,14 @@
     name: 'Genieo'
     url: 'http://www.genieo.com'
 
+- regex: 'GigablastOpenSource'
+  name: 'Gigablast'
+  category: 'Search bot'
+  url: 'https://github.com/gigablast/open-source-search-engine'
+  producer:
+    name: 'Matt Wells'
+    url: 'http://www.gigablast.com/faq.html'
+
 - regex: 'Gluten Free Crawler'
   name: 'Gluten Free Crawler'
   category: 'Crawler'
@@ -405,6 +557,18 @@
     name: 'NTT Resonant'
     url: 'http://goo.ne.jp'
 
+- regex: 'Google Favicon'
+  name: 'Google Favicon'
+  category: 'Crawler'
+
+- regex: 'Google Search Console'
+  name: 'Google Search Console'
+  category: 'Crawler'
+  url: 'https://search.google.com/search-console/about'
+  producer:
+    name: 'Google Inc.'
+    url: 'http://www.google.com'
+
 - regex: 'Google Page Speed Insights'
   name: 'Google PageSpeed Insights'
   category: 'Site Monitor'
@@ -421,6 +585,30 @@
     name: 'Google Inc.'
     url: 'http://www.google.com'
 
+- regex: 'Google-Cloud-Scheduler'
+  name: 'Google Cloud Scheduler'
+  category: 'Crawler'
+  url: 'https://cloud.google.com/scheduler'
+  producer:
+    name: 'Google Inc.'
+    url: 'https://www.google.com'
+
+- regex: 'Google-Structured-Data-Testing-Tool'
+  name: 'Google Structured Data Testing Tool'
+  category: 'Validator'
+  url: 'https://search.google.com/structured-data/testing-tool'
+  producer:
+    name: 'Google Inc.'
+    url: 'http://www.google.com'
+
+- regex: 'GoogleStackdriverMonitoring'
+  name: 'Google Stackdriver Monitoring'
+  category: 'Site Monitor'
+  url: 'https://cloud.google.com/monitoring'
+  producer:
+    name: 'Google Inc.'
+    url: 'https://www.google.com'
+
 - regex: 'via ggpht\.com GoogleImageProxy'
   name: 'Gmail Image Proxy'
   category: 'Crawler'
@@ -429,7 +617,39 @@
     name: 'Google Inc.'
     url: 'http://www.google.com'
 
-- regex: 'Googlebot(-Mobile|-Image|-Video|-News)?|Feedfetcher-Google|Google-Test|Google-Site-Verification|Google Web Preview|AdsBot-Google(-Mobile)?|Mediapartners-Google|Google.*/\+/web/snippet|GoogleProducer|Google[ -]Publisher[ -]Plugin'
+- regex: 'SeznamEmailProxy'
+  name: 'Seznam Email Proxy'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'Seznam.cz, a.s.'
+    url: 'http://www.seznam.cz/'
+
+- regex: 'Seznam-Zbozi-robot'
+  name: 'Seznam Zbozi.cz'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'Seznam.cz, a.s.'
+    url: 'https://www.zbozi.cz/'
+
+- regex: 'Heurekabot-Feed'
+  name: 'Heureka Feed'
+  category: 'Crawler'
+  url: 'https://sluzby.heureka.cz/napoveda/heurekabot/'
+  producer:
+    name: 'Heureka.cz, a.s.'
+    url: 'https://www.heureka.cz/'
+
+- regex: 'ShopAlike'
+  name: 'ShopAlike'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'Visual Meta'
+    url: 'https://www.shopalike.cz/'
+
+- regex: 'AdsBot-Google(-Mobile)?|Adwords-(DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(Adwords|AMPHTML|Assess|HotelAdsVerifier|Read-Aloud|Shopping-Quality|Site-Verification|speakr|Test|Youtube-Links)|(APIs|DuplexWeb|Feedfetcher|Mediapartners)-Google|Googlebot(-Mobile|-Image|-Video|-News)?|GoogleProducer|Google.*/\+/web/snippet'
   name: 'Googlebot'
   category: 'Search bot'
   url: 'http://www.google.com/bot.html'
@@ -445,6 +665,14 @@
     name: 'The Internet Archive'
     url: 'http://www.archive.org'
 
+- regex: 'HubSpot '
+  name: 'HubSpot'
+  category: 'Crawler'
+  producer:
+    name: 'HubSpot Inc.'
+    url: 'https://www.hubspot.com'
+
+
 - regex: 'HTTPMon'
   name: 'HTTPMon'
   category: 'Site Monitor'
@@ -461,6 +689,11 @@
     name: ''
     url: ''
 
+- regex: 'inoreader.com'
+  name: 'inoreader'
+  category: 'Feed Reader'
+  url: 'https://www.inoreader.com'
+
 - regex: 'iisbot'
   name: 'IIS Site Analysis'
   category: 'Crawler'
@@ -469,6 +702,21 @@
     name: 'Microsoft Corporation'
     url: 'http://www.microsoft.com'
 
+- regex: 'ips-agent'
+  name: 'IPS Agent'
+  category: 'crawler'
+  producer:
+    name: 'VeriSign, Inc'
+    url: 'http://www.verisign.com/'
+
+- regex: 'IP-Guide\.com'
+  name: 'IP-Guide Crawler'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: ''
+    url: 'https://ip-guide.com'
+
 - regex: 'kouio'
   name: 'Kouio'
   url: 'http://kouio.com/'
@@ -485,6 +733,14 @@
     name: ''
     url: ''
 
+- regex: '([A-z0-9]*)-Lighthouse'
+  name: 'Lighthouse'
+  category: 'Site Monitor'
+  url: 'https://developers.google.com/web/tools/lighthouse'
+  producer:
+    name: 'Lighthouse'
+    url: 'https://developers.google.com/web/tools/lighthouse'
+
 - regex: 'linkdexbot(-mobile)?|linkdex\.com'
   name: 'Linkdex Bot'
   category: 'Search bot'
@@ -532,6 +788,18 @@
     name: ''
     url: ''
 
+- regex : 'masscan'
+  name: 'masscan'
+  url: 'https://github.com/robertdavidgraham/masscan'
+  category: 'Crawler'
+  producer:
+    name: 'Robert Graham'
+    url: 'https://github.com/robertdavidgraham'
+
+- regex: 'Mastodon/'
+  name: 'Mastodon Bot'
+  category: 'Social Media Agent'
+
 - regex: 'meanpathbot'
   name: 'Meanpath Bot'
   category: 'Search bot'
@@ -540,6 +808,19 @@
     name: 'Meanpath'
     url: 'http://www.meanpath.com'
 
+- regex: 'MetaJobBot'
+  name: 'MetaJobBot'
+  category: 'Crawler'
+  url: 'http://www.metajob.at/the/crawler'
+  producer:
+    name: 'MetaJob'
+    url: 'http://www.metajob.at'
+
+- regex: 'MetaInspector'
+  name: 'MetaInspector'
+  category: 'Crawler'
+  url: 'https://github.com/jaimeiniesta/metainspector'
+
 - regex: 'MixrankBot'
   name: 'Mixrank Bot'
   category: 'Crawler'
@@ -556,6 +837,13 @@
     name: 'Majestic-12'
     url: 'http://majestic12.co.uk'
 
+- regex: 'Mnogosearch'
+  name: 'Mnogosearch'
+  category: 'Search bot'
+  url: 'http://www.mnogosearch.org/'
+  producer:
+    name: 'Lavtech.Com Corp.'
+    url: ''
 - regex: 'MojeekBot'
   name: 'MojeekBot'
   category: 'Search bot'
@@ -564,6 +852,14 @@
     name: 'Mojeek Ltd.'
     url: 'http://www.mojeek.com'
 
+- regex: 'munin'
+  name: 'Munin'
+  category: 'Site Monitor'
+  url: 'http://munin-monitoring.org/'
+  producer:
+    name: 'Munin'
+    url: 'http://munin-monitoring.org/'
+
 - regex: 'NalezenCzBot'
   name: 'NalezenCzBot'
   category: 'Crawler'
@@ -572,7 +868,19 @@
     name: 'Jaroslav Kuboš'
     url: ''
 
-- regex: 'Netcraft Web Server Survey'
+- regex: 'check_http/v'
+  name: 'Nagios check_http'
+  category: 'Site Monitor'
+  url: 'https://nagios.org'
+  producer:
+    name: 'Nagios Plugins Development Team'
+    url: 'https://nagios.org'
+
+- regex: 'nbertaupete95\(at\)gmail.com'
+  name: 'nbertaupete95'
+  category: 'Crawler'
+
+- regex: 'Netcraft( Web Server Survey| SSL Server Survey|SurveyAgent)'
   name: 'Netcraft Survey Bot'
   category: 'Search bot'
   url: ''
@@ -580,6 +888,14 @@
     name: 'Netcraft'
     url: 'http://www.netcraft.com'
 
+- regex: 'netEstate NE Crawler'
+  name: 'netEstate'
+  category: 'Crawler'
+  url: 'http://www.website-datenbank.de/Impressum'
+  producer:
+    name: 'netEstate GmbH'
+    url: 'https://www.netestate.de/en/'
+
 - regex: 'Netvibes'
   name: 'Netvibes'
   url: 'http://www.netvibes.com/'
@@ -612,7 +928,25 @@
     name: 'Northern Light'
     url: 'http://northernlight.com'
 
-- regex: 'omgilibot'
+- regex: 'Nmap Scripting Engine'
+  name: 'Nmap'
+  category: 'Security Checker'
+  url: 'https://nmap.org/book/nse.html'
+  producer:
+    name: 'Nmap'
+    url: 'https://nmap.org/'
+
+- regex: 'Nuzzel'
+  name: 'Nuzzel'
+  category: 'Crawler'
+  producer:
+    name: 'Nuzzel'
+    url: https://www.nuzzel.com/ 
+
+- regex: 'Octopus [0-9]'
+  name: 'Octopus'
+
+- regex: 'omgili(?:bot)?'
   name: 'Omgili bot'
   category: 'Search bot'
   url: 'http://www.omgili.com/Crawler.html'
@@ -660,6 +994,11 @@
     name: 'Smallrivers SA'
     url: 'http://www.paper.li'
 
+- regex: 'phantomas/'
+  name: 'Phantomas'
+  category: 'Site Monitor'
+  url: 'https://github.com/macbre/phantomas'
+
 - regex: 'phpservermon'
   name: 'PHP Server Monitor'
   category: 'Site Monitor'
@@ -668,6 +1007,22 @@
     name: 'PHP Server Monitor'
     url: 'http://www.phpservermonitor.org/'
 
+- regex: 'PocketParser'
+  name: 'PocketParser'
+  category: 'Read-it-later Service'
+  url: 'https://getpocket.com/pocketparser_ua'
+  producer:
+    name: 'Pocket'
+    url: 'https://getpocket.com/'
+
+- regex: 'PritTorrent'
+  name: 'PritTorrent'
+  category: 'Crawler'
+  url: 'https://github.com/astro/prittorrent'
+  producer:
+    name: 'Bitlove'
+    url: 'http://bitlove.org/'
+
 - regex: 'psbot(-page)?'
   name: 'Picsearch bot'
   category: 'Search bot'
@@ -684,6 +1039,22 @@
     name: 'Pingdom AB'
     url: 'https://www.pingdom.com'
 
+- regex: 'Quora Link Preview'
+  name: 'Quora Link Preview'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'Quora'
+    url: 'http://www.quora.com'
+
+- regex: 'RamblerMail'
+  name: 'RamblerMail Image Proxy'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'Rambler&Co'
+    url: 'https://rambler-co.ru/'
+
 - regex: 'QuerySeekerSpider'
   name: 'QuerySeekerSpider'
   category: 'Crawler'
@@ -692,6 +1063,19 @@
     name: 'QueryEye Inc.'
     url: 'http://queryeye.com'
 
+- regex: 'Qwantify'
+  name: 'Qwantify'
+  category: 'Crawler'
+  url: 'https://www.qwant.com/'
+  producer:
+    name: 'Qwant Corporation'
+    url: 'https://www.qwant.com/'
+
+- regex: 'Rainmeter'
+  name: 'Rainmeter'
+  category: 'Crawler'
+  url: 'https://www.rainmeter.net'
+
 - regex: 'redditbot'
   name: 'Reddit Bot'
   category: 'Social Media Agent'
@@ -700,6 +1084,14 @@
     name: 'reddit inc.'
     url: 'http://www.reddit.com'
 
+- regex: 'Riddler'
+  name: 'Riddler'
+  category: 'Security search bot'
+  url: 'https://riddler.io/about'
+  producer:
+    name: 'F-Secure'
+    url: 'https://www.f-secure.com'
+
 - regex: 'rogerbot'
   name: 'Rogerbot'
   category: 'Crawler'
@@ -716,6 +1108,14 @@
     name: 'Roihunter a.s.'
     url: 'http://roihunter.com/'
 
+- regex: 'SafeDNSBot'
+  name: 'SafeDNSBot'
+  category: 'Crawler'
+  url: 'https://www.safedns.com/searchbot'
+  producer:
+    name: 'SafeDNS, Inc.'
+    url: 'https://www.safedns.com/'
+
 - regex: 'Scrapy'
   name: 'Scrapy'
   category: 'Crawler'
@@ -761,6 +1161,19 @@
     name: 'SEO Engine'
     url: 'http://www.seoengine.com'
 
+- regex: 'SEOkicks-Robot'
+  name: 'SEOkicks-Robot'
+  category: 'Crawler'
+  url: 'http://www.seokicks.de/robot.html'
+  producer:
+    name: 'SEOkicks'
+    url: 'https://www.seokicks.de/'
+
+- regex: 'seoscanners\.net'
+  name: 'Seoscanners.net'
+  category: 'Crawler'
+  url: ''
+
 - regex: 'SkypeUriPreview'
   name: 'Skype URI Preview'
   category: 'Service Agent'
@@ -777,6 +1190,14 @@
     name: 'Seznam.cz, a.s.'
     url: 'http://www.seznam.cz/'
 
+- regex: 'shopify-partner-homepage-scraper'
+  name: 'Shopify Partner'
+  category: 'Crawler'
+  url: 'https://www.shopify.com/partners'
+  producer:
+    name: 'Shopify'
+    url: 'https://www.shopify.com/'
+
 - regex: 'ShopWiki'
   name: 'ShopWiki'
   category: 'Search tools'
@@ -809,6 +1230,27 @@
     name: 'SISTRIX GmbH'
     url: 'http://www.sistrix.de'
 
+- regex: 'SISTRIX Optimizer'
+  name: 'SISTRIX Optimizer'
+  category: 'Crawler'
+  url: 'https://optimizer.sistrix.com'
+  producer:
+    name: 'SISTRIX GmbH'
+    url: 'http://www.sistrix.de'
+
+- regex: 'SiteSucker'
+  name: 'SiteSucker'
+  category: 'Crawler'
+  url: 'http://ricks-apps.com/osx/sitesucker/'
+
+- regex: 'sixy.ch'
+  name: 'Sixy.ch'
+  category: 'Site Monitor'
+  url: 'http://sixy.ch'
+  producer:
+    name: 'Manuel Kasper'
+    url: 'https://neon1.net/'
+
 - regex: 'Slackbot|Slack-ImgProxy'
   name: 'Slackbot'
   category: 'Crawler'
@@ -833,6 +1275,30 @@
     name: 'Tencent Holdings'
     url: 'http://www.soso.com'
 
+- regex: 'sqlmap/'
+  name: 'sqlmap'
+  category: 'Security Checker'
+  url: 'http://sqlmap.org/'
+  producer:
+    name: 'sqlmap'
+    url: 'http://sqlmap.org/'
+
+- regex: 'SSL Labs'
+  name: 'SSL Labs'
+  category: 'Validator'
+  url: 'https://www.ssllabs.com/about/assessment.html'
+  producer:
+    name: 'SSL Labs'
+    url: 'https://www.ssllabs.com/about/assessment.html'
+
+- regex: 'StatusCake'
+  name: 'StatusCake'
+  category: 'Site Monitor'
+  url: 'https://www.statuscake.com'
+  producer:
+    name: 'StatusCake'
+    url: 'https://www.statuscake.com'
+
 - regex: 'Superfeedr bot'
   name: 'Superfeedr Bot'
   category: 'Feed Fetcher'
@@ -841,6 +1307,11 @@
     name: 'Superfeedr'
     url: 'https://superfeedr.com/'
 
+- regex: 'Sparkler/[0-9]'
+  name: 'Sparkler'
+  category: 'Crawler'
+  url: 'https://github.com/USCDataScience/sparkler'
+
 - regex: 'Spinn3r'
   name: 'Spinn3r'
   category: 'Crawler'
@@ -865,10 +1336,23 @@
     name: 'Domain Tools'
     url: 'http://www.domaintools.com'
 
+- regex: 'TarmotGezgin'
+  name: 'Tarmot Gezgin'
+  url: 'http://www.tarmot.com/gezgin/'
+  category: 'Search bot'
+
 - regex: 'TelegramBot'
-  name: 'TelgramBot'
+  name: 'TelegramBot'
   url: 'https://telegram.org/blog/bot-revolution'
 
+- regex: 'TLSProbe'
+  name: 'TLSProbe'
+  url: 'https://scan.trustnet.venafi.com/'
+  category: 'Security search bot'
+  producer:
+    name: 'Venafi TrustNet'
+    url: 'https://www.venafi.com'
+
 - regex: 'TinEye-bot'
   name: 'TinEye Crawler'
   category: 'Search bot'
@@ -885,6 +1369,19 @@
     name: ''
     url: ''
 
+- regex: 'theoldreader.com'
+  name: 'theoldreader'
+  category: 'Feed Reader'
+  url: 'https://theoldreader.com'
+
+- regex: 'trendictionbot'
+  name: 'Trendiction Bot'
+  category: 'Crawler'
+  url: 'http://www.trendiction.de/bot'
+  producer:
+    name: 'Talkwalker Inc.'
+    url: 'http://www.talkwalker.com'
+
 - regex: 'TurnitinBot'
   name: 'TurnitinBot'
   category: 'Crawler'
@@ -909,6 +1406,13 @@
     name: 'Mediasift'
     url: ''
 
+- regex: 'Twingly Recon'
+  name: 'Twingly Recon'
+  category: 'Crawler'
+  producer:
+    name: 'Twingly'
+    url: 'https://www.twingly.com'
+
 - regex: 'Twitterbot'
   name: 'Twitterbot'
   category: 'Social Media Agent'
@@ -917,6 +1421,30 @@
     name: 'Twitter'
     url: 'http://www.twitter.com'
 
+- regex: 'UniversalFeedParser'
+  name: 'UniversalFeedParser'
+  category: 'Feed Fetcher'
+  url: 'https://github.com/kurtmckee/feedparser'
+  producer:
+    name: 'Kurt McKee'
+    url: 'https://github.com/kurtmckee'
+
+- regex: 'via secureurl\.fwdcdn\.com'
+  name: 'UkrNet Mail Proxy'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'UkrNet Ltd'
+    url: 'https://www.ukr.net/'
+
+- regex: 'Uptimebot'
+  name: 'Uptimebot'
+  category: 'Site Monitor'
+  url: 'https://uptime.com/uptimebot'
+  producer:
+    name: 'Uptime'
+    url: 'https://uptime.com'
+
 - regex: 'UptimeRobot'
   name: 'Uptime Robot'
   category: 'Site Monitor'
@@ -933,6 +1461,22 @@
     name: 'Profound Networks'
     url: 'http://www.profound.net'
 
+- regex: 'Vagabondo'
+  name: 'Vagabondo'
+  category: 'Crawler'
+  url: ''
+  producer:
+    name: 'WiseGuys'
+    url: 'http://www.wise-guys.nl/'
+
+- regex: 'vkShare; '
+  name: 'VK Share Button'
+  category: 'Crawler'
+  url: 'http://vk.com/dev/Share'
+  producer:
+    name: 'VK'
+    url: 'http://vk.com/'
+
 - regex: 'VSMCrawler'
   name: 'Visual Site Mapper Crawler'
   category: 'Crawler'
@@ -965,7 +1509,7 @@
     name: 'W3C'
     url: 'http://www.w3.org'
 
-- regex: 'W3C_Validator'
+- regex: 'W3C_Validator|Validator.nu'
   name: 'W3C Markup Validation Service'
   category: 'Validator'
   url: 'http://validator.w3.org/services'
@@ -989,6 +1533,18 @@
     name: 'W3C'
     url: 'http://www.w3.org'
 
+- regex: 'Wappalyzer'
+  name: 'Wappalyzer'
+  url: 'https://github.com/AliasIO/Wappalyzer'
+  producer:
+    name: 'AliasIO'
+    url: 'https://github.com/AliasIO'
+
+- regex: 'PTST/'
+  name: 'WebPageTest'
+  category: 'Site Monitor'
+  url: 'https://www.webpagetest.org'
+
 - regex: 'WeSEE(:Search)?'
   name: 'WeSEE:Search'
   category: 'Search bot'
@@ -1013,6 +1569,14 @@
     name: 'WebSitePulse'
     url: 'http://www.websitepulse.com/'
 
+- regex: 'WordPress'
+  name: 'WordPress'
+  category: 'Service Agent'
+  url: 'https://wordpress.org/'
+  producer:
+    name: 'Wordpress.org'
+    url: 'https://wordpress.org/'
+
 - regex: 'Wotbox'
   name: 'Wotbox'
   category: 'Search bot'
@@ -1021,6 +1585,14 @@
     name: 'Wotbox'
     url: 'http://www.wotbox.com'
 
+- regex: 'XenForo'
+  name: 'XenForo'
+  category: 'Service Agent'
+  url: 'https://xenforo.com/'
+  producer:
+    name: 'XenForo Ltd.'
+    url: 'https://xenforo.com/'
+
 - regex: 'yacybot'
   name: 'YaCy'
   category: 'Search bot'
@@ -1053,7 +1625,15 @@
     name: 'Yahoo! Inc.'
     url: 'http://www.yahoo.com'
 
-- regex: 'Yandex(Bot|Images|Antivirus|Direct|Blogs|Favicons|ImageResizer|News(links)?|Metrika|\.Gazeta Bot)|YaDirectFetcher'
+- regex: 'Y!J-BRW'
+  name: 'Yahoo! Japan BRW'
+  category: 'Crawler'
+  url: 'https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/ウェブページにアクセスするシステムのユーザーエージェントについて'
+  producer:
+    name: 'Yahoo! Japan Corp.'
+    url: 'https://www.yahoo.co.jp/'
+
+- regex: 'Yandex(SpravBot|ScreenshotBot|MobileBot|AccessibilityBot|ForDomain|Vertis|Market|Catalog|Calendar|Sitelinks|AdNet|Pagechecker|Webmaster|Media|Video|Bot|Images|Antivirus|Direct|Blogs|Favicons|ImageResizer|Verticals|News(links)?|Metrika|\.Gazeta Bot)|YaDirectFetcher|YandexTurbo|YandexTracker|YandexSearchShop|YandexRCA|YandexPartner|YandexOntoDBAPI|YandexOntoDB|YandexMobileScreenShotBot'
   name: 'Yandex Bot'
   category: 'Search bot'
   url: 'http://www.yandex.com/bots'
@@ -1061,7 +1641,7 @@
     name: 'Yandex LLC'
     url: 'http://company.yandex.com'
 
-- regex: 'Yeti'
+- regex: 'Yeti|NaverJapan'
   name: 'Yeti/Naverbot'
   category: 'Search bot'
   url: 'http://help.naver.com/robots/'
@@ -1090,6 +1670,11 @@
     name: 'YunYun'
     url: 'http://www.yunyun.com'
 
+- regex: 'zgrab'
+  name: 'zgrab'
+  category: 'Security Checker'
+  url: 'https://github.com/zmap/zgrab'
+
 - regex: 'Zookabot'
   name: 'Zookabot'
   category: 'Crawler'
@@ -1138,9 +1723,9 @@
     name: 'HubPages'
     url: 'http://hubpages.com/'
 
-- regex: 'Pinterest/\d\.\d.*www\.pinterest\.com.*'
+- regex: 'Pinterest(bot)?/\d\.\d.*www\.pinterest\.com.*'
   name: 'Pinterest'
-  url: ''
+  url: 'http://www.pinterest.com/bot.html'
   category: 'Crawler'
   producer:
     name: 'Pinterest'
@@ -1154,6 +1739,30 @@
     name: 'Site24x7'
     url: 'https://www.site24x7.com'
 
+- regex: 's~snapchat-proxy'
+  name: 'Snapchat Proxy'
+  category: 'Crawler'
+  url: 'https://www.snapchat.com'
+  producer:
+    name: 'Snapchat Inc.'
+    url: 'https://www.snapchat.com'
+
+- regex: "Let's Encrypt validation server"
+  name: "Let's Encrypt Validation"
+  category: 'Service Agent'
+  url: 'https://letsencrypt.org/how-it-works/'
+  producer:
+    name: "Let's Encrypt"
+    url: 'https://letsencrypt.org'
+
+- regex: 'GrapeshotCrawler'
+  name: 'Grapeshot'
+  category: 'Crawler'
+  url: 'https://www.grapeshot.com/crawler'
+  producer:
+    name: 'Grapeshot'
+    url: 'https://www.grapeshot.com'
+
 - regex: 'www\.monitor\.us'
   name: 'Monitor.Us'
   category: 'Site Monitor'
@@ -1162,6 +1771,22 @@
     name: 'Monitor.Us'
     url: 'http://www.monitor.us'
 
+- regex: 'Catchpoint( bot)?'
+  name: 'Catchpoint'
+  category: 'Site Monitor'
+  url: ''
+  producer:
+    name: 'Catchpoint Systems'
+    url: 'http://www.catchpoint.com/'
+
+- regex: 'bitlybot'
+  name: 'BitlyBot'
+  category: 'Crawler'
+  url: 'https://bitly.com'
+  producer:
+    name: 'Bitly, Inc.'
+    url: 'https://bitly.com'
+
 - regex: 'Zao/'
   name: 'Zao'
   category: 'Crawler'
@@ -1217,9 +1842,204 @@
 - regex: 'Server Density Service Monitoring.*'
   name: 'Server Density'
 
-- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?! Build)|zeal|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Catchpoint bot|Google SketchUp|Read%20Later|Minimo|RackspaceBot)'
+- regex: 'RSSRadio \(Push Notification Scanner;support@dorada\.co\.uk\)'
+  name: 'RSSRadio Bot'
+
+- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com|tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?! Build)|zeal|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|RackspaceBot|robots|SeopultContentAnalyzer|7Siters|centuryb.o.t9)'
   name: 'Generic Bot'
 
+- regex: '^sentry'
+  name: 'Sentry Bot'
+  producer:
+    name: 'Sentry'
+    url: 'https://sentry.io'
+
+- regex: '^Spotify'
+  name: 'Spotify'
+  producer:
+    name: 'Spotify'
+    url: 'https://www.spotify.com'
+
+- regex: 'The Knowledge AI'
+  name: 'The Knowledge AI'
+  category: 'Crawler'
+
+- regex: 'Embedly'
+  name: 'Embedly'
+  category: 'Crawler'
+  url: 'https://support.embed.ly/hc/en-us'
+  producer:
+    name: 'A Medium, Corp.'
+    url: 'https://medium.com/'
+
+- regex: 'BrandVerity'
+  name: 'BrandVerity'
+  category: 'Crawler'
+  url: 'https://www.brandverity.com/why-is-brandverity-visiting-me'
+  producer:
+    name: 'BrandVerity, Inc.'
+    url: 'https://www.brandverity.com/'
+
+- regex: 'Kaspersky Lab CFR link resolver'
+  name: 'Kaspersky'
+  category: 'Security Checker'
+  url: 'https://www.kaspersky.com/'
+  producer:
+    name: 'AO Kaspersky Lab'
+    url: 'https://www.kaspersky.com/'
+
+- regex: 'eZ Publish Link Validator'
+  name: 'eZ Publish Link Validator'
+  category: 'Crawler'
+  url: 'https://ez.no/'
+  producer:
+    name: 'eZ Systems AS'
+    url: 'https://ez.no/'
+
+- regex: 'woorankreview'
+  name: 'WooRank'
+  category: 'Search bot'
+  url: 'https://www.woorank.com/'
+  producer:
+    name: 'WooRank sprl'
+    url: 'https://www.woorank.com/'
+
+- regex: '(Match|LinkCheck) by Siteimprove.com'
+  name: 'Siteimprove'
+  category: 'Search bot'
+  url: 'https://siteimprove.com/'
+  producer:
+    name: 'Siteimprove GmbH'
+    url: 'https://siteimprove.com/'
+
+- regex: 'CATExplorador'
+  name: 'CATExplorador'
+  category: 'Search bot'
+  url: 'https://fundacio.cat/ca/domini/'
+  producer:
+    name: 'Fundació puntCAT'
+    url: 'https://fundacio.cat/ca/domini/'
+
+- regex: 'Buck'
+  name: 'Buck'
+  category: 'Search bot'
+  url: 'https://hypefactors.com/'
+  producer:
+    name: 'Hypefactors A/S'
+    url: 'https://hypefactors.com/'
+
+- regex: 'tracemyfile'
+  name: 'TraceMyFile'
+  category: 'Search bot'
+  url: 'https://www.tracemyfile.com/'
+  producer:
+    name: 'Idee Inc.'
+    url: 'http://ideeinc.com/'
+
+- regex: 'zelist.ro feed parser'
+  name: 'Ze List'
+  url: 'https://www.zelist.ro/'
+  category: 'Feed Fetcher'
+  producer:
+    name: 'Treeworks SRL'
+    url: 'https://www.tree.ro/'
+
+- regex: 'weborama-fetcher'
+  name: 'Weborama'
+  category: 'Search bot'
+  url: 'https://weborama.com/'
+  producer:
+    name: 'Weborama SA'
+    url: 'https://weborama.com/'
+
+- regex: 'BoardReader Favicon Fetcher'
+  name: 'BoardReader'
+  category: 'Search bot'
+  url: 'http://boardreader.com/'
+  producer:
+    name: 'Effyis Inc'
+    url: 'http://boardreader.com/'
+
+- regex: 'IDG/IT'
+  name: 'IDG/IT'
+  category: 'Search bot'
+  url: 'https://spaziodati.eu/'
+  producer:
+    name: 'SpazioDati S.r.l.'
+    url: 'https://spaziodati.eu/'
+
+- regex: 'Bytespider'
+  name: 'Bytespider'
+  category: 'Search bot'
+  url: 'https://bytedance.com/'
+  producer:
+    name: 'ByteDance Ltd.'
+    url: 'https://bytedance.com/'
+
+- regex: 'WikiDo'
+  name: 'WikiDo'
+  category: 'Search bot'
+  url: 'https://www.wikido.com/'
+  producer:
+    name: 'Fotolitografie Fiorentine di Becchi Antonio s.n.c.'
+    url: 'https://www.wikido.com/'
+
+- regex: 'AwarioSmartBot'
+  name: 'Awario'
+  category: 'Search bot'
+  url: 'https://awario.com/bots.html'
+  producer:
+    name: 'Awario'
+    url: 'https://awario.com/'
+
+- regex: 'AwarioRssBot'
+  name: 'Awario'
+  category: 'Feed Fetcher'
+  url: 'https://awario.com/bots.html'
+  producer:
+    name: 'Awario'
+    url: 'https://awario.com/'
+
+- regex: 'oBot'
+  name: 'oBot'
+  category: 'Search bot'
+  url: 'http://www.xforce-security.com/crawler/'
+  producer:
+    name: 'IBM Germany Research & Development GmbH'
+    url: 'https://exchange.xforce.ibmcloud.com/'
+
+- regex: 'SMTBot'
+  name: 'SMTBot'
+  category: 'Search bot'
+  url: 'https://www.similartech.com/smtbot'
+  producer:
+    name: 'SimilarTech Ltd.'
+    url: 'https://www.similartech.com/'
+
+- regex: 'LCC'
+  name: 'LCC'
+  category: 'Search bot'
+  url: 'https://corpora.uni-leipzig.de/crawler_faq.html'
+  producer:
+    name: 'Universität Leipzig'
+    url: 'https://www.uni-leipzig.de/'
+
+- regex: 'Startpagina-Linkchecker'
+  name: 'Startpagina Linkchecker'
+  category: 'Search bot'
+  url: 'https://www.startpagina.nl/linkchecker'
+  producer:
+    name: 'Startpagina B.V.'
+    url: 'https://www.startpagina.nl/'
+
+- regex: 'GTmetrix'
+  name: 'GTmetrix'
+  category: 'Crawler'
+  url: 'https://gtmetrix.com/'
+  producer:
+    name: 'Carbon60 Operating Co. Ltd.'
+    url: 'https://www.carbon60.com/'
+
 # Generic detections
 
 - regex: 'Nutch'
@@ -1230,5 +2050,5 @@
     name: 'The Apache Software Foundation'
     url: 'http://www.apache.org/foundation/'
 
-- regex: '[a-z0-9\-_]*((?<!cu)bot|crawler|archiver|transcoder|spider)([^a-z]|$)'
+- regex: '[a-z0-9\-_]*((?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9])|crawler|crawl|checker|archiver|transcoder|spider)([^a-z]|$)'
   name: 'Generic Bot'