device_detector 1.1.0 → 1.1.1

data/lib/device_detector/parser.rb

@@ -58,7 +58,29 @@ class DeviceDetector
     end
 
     def load_regexes(file_paths)
-      file_paths.map { |path, full_path| [path, symbolize_keys!(YAML.load_file(full_path))] }
+      file_paths.map do |path, full_path|
+        object = YAML.load_file(full_path)
+        object = rewrite_device_object!(object) if is_device_yml_file?(full_path)
+        object = rewrite_vendor_object!(object) if is_vendor_yml_file?(full_path)
+
+        [path, symbolize_keys!(object)]
+      end
+    end
+
+    def is_device_yml_file?(file_path)
+      file_path.include?('/regexes/device/')
+    end
+
+    def is_vendor_yml_file?(file_path)
+      file_path.include?('/regexes/vendorfragments')
+    end
+
+    def rewrite_vendor_object!(object)
+      object.map { |key, values| values.map { |v| { 'regex_name' => key, 'regex' => v } } }.flatten
+    end
+
+    def rewrite_device_object!(object)
+      object.map { |key, value| [key, { 'regex_name' => key }.merge!(value)] }.to_h
     end
 
     def symbolize_keys!(object)
@@ -88,8 +110,8 @@ class DeviceDetector
       Regexp.new('(?:^|[^A-Z0-9\-_]|[^A-Z0-9\-]_|sprd-|MZ-)(?:' + src + ')', Regexp::IGNORECASE)
     end
 
-    def from_cache(key)
-      DeviceDetector.cache.get_or_set(key) { yield }
+    def from_cache(key, &block)
+      DeviceDetector.cache.get_or_set(key, &block)
     end
   end
 end

data/lib/device_detector/vendor_fragment.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'set'
+
+class DeviceDetector
+  class VendorFragment < Parser
+    def name
+      vendor_fragment_info
+    end
+
+    private
+
+    def vendor_fragment_info
+      from_cache(['vendor_fragment', self.class.name, user_agent]) do
+        return if regex_meta.nil? || regex_meta.empty?
+
+        regex_meta[:regex_name]
+      end
+    end
+
+    def filenames
+      ['vendorfragments.yml']
+    end
+  end
+end

data/lib/device_detector/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class DeviceDetector
-  VERSION = '1.1.0'
+  VERSION = '1.1.1'
 end

data/lib/device_detector.rb

@@ -15,13 +15,34 @@ require 'device_detector/device'
 require 'device_detector/os'
 require 'device_detector/browser'
 require 'device_detector/client_hint'
+require 'device_detector/vendor_fragment'
 
 class DeviceDetector
   attr_reader :client_hint, :user_agent
 
   def initialize(user_agent, headers = nil)
     @client_hint = ClientHint.new(headers)
-    @user_agent = user_agent
+    utf8_user_agent = encode_user_agent_if_needed(user_agent)
+    @user_agent = set_user_agent(utf8_user_agent)
+  end
+
+  # https://github.com/matomo-org/device-detector/blob/c235832dba13961ab0f71b681616baf1aa48de23/Parser/Device/AbstractDeviceParser.php#L1873
+  def set_user_agent(user_agent)
+    return user_agent if client_hint.model.nil?
+
+    regex = build_regex('Android 10[.\d]*; K(?: Build/|[;)])')
+    return user_agent unless user_agent =~ regex
+
+    version = client_hint.os_version || '10'
+
+    user_agent.gsub(regex, "Android #{version}, #{client_hint.model}")
+  end
+
+  def encode_user_agent_if_needed(user_agent)
+    return if user_agent.nil?
+    return user_agent if user_agent.encoding.name == 'UTF-8'
+
+    user_agent.encode('utf-8', 'binary', undef: :replace)
   end
 
   def name
@@ -53,13 +74,18 @@ class DeviceDetector
   end
 
   def device_name
+    return if fake_ua?
+
     device.name || client_hint.model || fix_for_x_music
   end
 
   def device_brand
+    return if fake_ua?
+
     # Assume all devices running iOS / Mac OS are from Apple
     brand = device.brand
     brand = 'Apple' if brand.nil? && ['Apple TV', 'iOS', 'Mac'].include?(os_name)
+
     brand
   end
 
@@ -206,6 +232,11 @@ class DeviceDetector
     @os ||= OS.new(user_agent)
   end
 
+  # https://github.com/matomo-org/device-detector/blob/827a3fab7e38c3274c18d2f5f5bc2a78b7ef4a3a/DeviceDetector.php#L921C5-L921C5
+  def fake_ua?
+    os_name == 'Android' && device.brand == 'Apple'
+  end
+
   # https://github.com/matomo-org/device-detector/blob/be1c9ef486c247dc4886668da5ed0b1c49d90ba8/Parser/Client/Browser.php#L772
   # Fix mobile browser names e.g. Chrome => Chrome Mobile
   def mobile_fix?
@@ -218,7 +249,7 @@ class DeviceDetector
 
   # Related to issue mentionned in device.rb#1562
   def fix_for_x_music
-    user_agent.include?('X-music Ⅲ') ? 'X-Music III' : nil
+    user_agent&.include?('X-music Ⅲ') ? 'X-Music III' : nil
   end
 
   def skip_os_version?

data/regexes/bots.yml

@@ -13,6 +13,14 @@
     name: 'Plesk International GmbH'
     url: 'https://www.plesk.com'
 
+- regex: 'Cloudflare-Healthchecks'
+  name: 'Cloudflare Health Checks'
+  category: 'Service Agent'
+  url: 'https://developers.cloudflare.com/health-checks/'
+  producer:
+    name: 'CloudFlare'
+    url: 'http://www.cloudflare.com'
+
 - regex: '360Spider'
   name: '360Spider'
   category: 'Search bot'
@@ -704,7 +712,7 @@
     name: 'Visual Meta'
     url: 'https://www.shopalike.cz/'
 
-- regex: 'AdsBot-Google|Adwords-(DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|HotelAdsVerifier|Read-Aloud|Shopping-Quality|Site-Verification|speakr|Stale-Content-Probe|Test|Youtube-Links)|(APIs|DuplexWeb|Feedfetcher|Mediapartners)-Google|Googlebot|Google(?:AdSenseInfeed|AssociationService|Producer)|Google.*/\+/web/snippet'
+- regex: 'AdsBot-Google|Adwords-(DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|HotelAdsVerifier|Read-Aloud|Shopping-Quality|Site-Verification|speakr|Stale-Content-Probe|Test|Youtube-Links)|(APIs|DuplexWeb|Feedfetcher|Mediapartners)-Google|Googlebot|Google(?:AdSenseInfeed|AssociationService|Prober|Producer)|Google.*/\+/web/snippet'
   name: 'Googlebot'
   category: 'Search bot'
   url: 'http://www.google.com/bot.html'
@@ -727,6 +735,11 @@
     name: 'HubSpot Inc.'
     url: 'https://www.hubspot.com'
 
+- regex: 'vuhuvBot'
+  name: 'Vuhuv Bot'
+  category: 'Crawler'
+  url: 'http://vuhuv.com/bot.html'
+
 - regex: 'HTTPMon'
   name: 'HTTPMon'
   category: 'Site Monitor'
@@ -1020,6 +1033,14 @@
 - regex: 'Octopus [0-9]'
   name: 'Octopus'
 
+- regex: 'OnlineOrNot.com_bot'
+  name: 'OnlineOrNot Bot'
+  category: 'Site Monitor'
+  url: 'https://onlineornot.com/website-monitoring'
+  producer:
+    name: 'OnlineOrNot'
+    url: 'https://onlineornot.com'
+
 - regex: 'omgili'
   name: 'Omgili bot'
   category: 'Search bot'
@@ -2025,7 +2046,15 @@
     name: 'WooRank sprl'
     url: 'https://www.woorank.com/'
 
-- regex: '(Match|LinkCheck) by Siteimprove.com'
+- regex: 'by Siteimprove\.com'
+  name: 'Siteimprove'
+  category: 'Search bot'
+  url: 'https://siteimprove.com/'
+  producer:
+    name: 'Siteimprove GmbH'
+    url: 'https://siteimprove.com/'
+
+- regex: 'Image size by Siteimprove\.com'
   name: 'Siteimprove'
   category: 'Search bot'
   url: 'https://siteimprove.com/'
@@ -2153,6 +2182,14 @@
     name: 'Startpagina B.V.'
     url: 'https://www.startpagina.nl/'
 
+- regex: 'MoodleBot-Linkchecker'
+  name: 'MoodleBot Linkchecker'
+  category: 'Search bot'
+  url: 'hhttps://docs.moodle.org/en/Usage'
+  producer:
+    name: 'Moodle Pty Ltd'
+    url: 'https://moodle.org/'
+
 - regex: 'GTmetrix'
   name: 'GTmetrix'
   category: 'Crawler'
@@ -2412,6 +2449,13 @@
   producer:
     name: 'Hatena Co., Ltd.'
     url: 'https://www.hatena.ne.jp'
+- regex: 'Hatena-?Bookmark'
+  name: 'Hatena Bookmark'
+  category: 'Crawler'
+  url: 'https://www.hatena.ne.jp/faq/'
+  producer:
+    name: 'Hatena Co., Ltd.'
+    url: 'https://www.hatena.ne.jp'
 
 - regex: 'RyowlEngine/(\d+)'
   name: 'Ryowl'
@@ -2556,6 +2600,14 @@
   category: 'Security Checker'
   url: 'https://github.com/LeakIX/l9explore'
 
+- regex: 'l9scan/|^Lkx-(.*)/([\d+.]+)'
+  name: 'LeakIX'
+  category: 'Security Checker'
+  url: 'https://leakix.net/'
+  producer:
+    name: 'BaDaaS SRL'
+    url: 'https://leakix.net/'
+
 - regex: 'MegaIndex.ru/([\d+\.])'
   name: 'MegaIndex'
   category: 'Crawler'
@@ -3047,14 +3099,6 @@
     name: 'New Work SE'
     url: 'https://www.xing.com/'
 
-- regex: '^Lkx-(.*)/([\d+.]+)'
-  name: 'LeakIX'
-  category: 'Security Checker'
-  url: 'https://leakix.net/'
-  producer:
-    name: 'BaDaaS SRL'
-    url: 'https://leakix.net/'
-
 - regex: 'RepoLookoutBot/([\d+.]+)'
   name: 'Repo Lookout'
   category: 'Security Checker'
@@ -3217,6 +3261,124 @@
   category: 'Crawler'
   url: 'https://reqbin.com/curl'
 
+- regex: 'XoviBot/([\d+.]+)'
+  name: 'XoviBot'
+  category: 'Crawler'
+  url: 'https://www.xovibot.net'
+  producer:
+    name: 'Xovi GmbH'
+    url: 'http://www.xovi.de'
+
+- regex: 'Overcast/([\d+.]+) Podcast Sync'
+  name: 'Overcast Podcast Sync'
+  category: 'Service Agent'
+  url: 'https://overcast.fm/podcasterinfo'
+
+- regex: '^Verity/([\d+.]+)'
+  name: 'GumGum Verity'
+  category: 'Service Agent'
+  url: 'https://gumgum.com/verity'
+
+- regex: 'hackermention'
+  name: 'hackermention'
+  category: 'Feed Reader'
+  url: 'https://github.com/snarfed/hackermention'
+
+- regex: 'BitSightBot/([\d+.]+)'
+  name: 'BitSight'
+  category: 'Security Checker'
+  url: 'https://www.bitsight.com/'
+  producer:
+    name: 'BitSight Technologies, Inc.'
+    url: 'https://www.bitsight.com/'
+
+- regex: 'Ezgif/([\d+.]+)'
+  name: 'Ezgif'
+  category: 'Service Agent'
+  url: 'https://ezgif.com/about'
+
+- regex: 'intelx.io_bot'
+  name: 'Intelligence X'
+  category: 'Crawler'
+  url: 'https://intelx.io/'
+  producer:
+    name: 'Kleissner Investments s.r.o.'
+    url: 'https://intelx.io/'
+
+- regex: 'FemtosearchBot/([\d+.]+)'
+  name: 'Femtosearch'
+  category: 'Crawler'
+  url: 'http://femtosearch.com/'
+  producer:
+    name: 'Grier Forensics, LLC'
+    url: 'https://www.grierforensics.com/'
+
+- regex: 'AdsTxtCrawler/([\d+.]+)'
+  name: 'AdsTxtCrawler'
+  category: 'Crawler'
+  url: 'https://github.com/InteractiveAdvertisingBureau/adstxtcrawler'
+  producer:
+    name: 'IAB Technology Laboratory, Inc.'
+    url: 'https://iabtechlab.com/'
+
+- regex: 'Morningscore'
+  name: 'Morningscore Bot'
+  category: 'Crawler'
+  url: 'https://morningscore.io/'
+  producer:
+    name: 'Morningscore'
+    url: 'https://morningscore.io/'
+
+- regex: 'Uptime-Kuma/([\d+.]+)'
+  name: 'Uptime-Kuma'
+  category: 'Site Monitor'
+  url: 'https://github.com/louislam/uptime-kuma'
+
+- regex: 'ChatGPT-User'
+  name: 'ChatGPT'
+  category: 'Crawler'
+  url: 'https://platform.openai.com/docs/plugins/bot'
+  producer:
+    name: 'OpenAI OpCo, LLC'
+    url: 'https://openai.com/'
+
+- regex: 'BrightEdge Crawler/([\d+.]+)'
+  name: 'BrightEdge'
+  category: 'Crawler'
+  url: 'https://www.brightedge.com/'
+  producer:
+    name: 'BrightEdge Technologies, Inc'
+    url: 'https://www.brightedge.com/'
+
+- regex: 'sfFeedReader/([\d+.]+)'
+  name: 'sfFeedReader'
+  url: 'https://github.com/diem-project/sfFeed2Plugin'
+  category: 'Feed Fetcher'
+
+- regex: 'cyberscan.io'
+  name: 'Cyberscan'
+  category: 'Security Checker'
+  url: 'https://www.cyberscan.io/'
+  producer:
+    name: 'DGC Verwaltungs GmbH'
+    url: 'https://dgc.org/'
+
+- regex: 'deepcrawl\.com'
+  name: 'Lumar'
+  category: 'Crawler'
+  url: 'https://deepcrawl.com/bot'
+  producer:
+    name: 'Lumar'
+    url: 'https://www.lumar.io/'
+
+- regex: 'RepoLookoutBot'
+  name: 'Repo Lookout'
+  category: 'Crawler'
+  url: 'https://www.repo-lookout.org/'
+  producer:
+    name: 'Crissy Field GmbH'
+    url: 'https://www.crissyfield.de/'
+
 # Generic detections
 - regex: '[a-z0-9\-_]*((?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9]|[ _]Senior|[ _]Junior)|crawler|crawl|checker|archiver|transcoder|spider)([^a-z]|$)'
   name: 'Generic Bot'

data/regexes/client/browsers.yml

@@ -319,7 +319,7 @@
     default: 'Blink'
 
 # Chromium GOST (https://github.com/deemru/chromium-gost)
-- regex: 'Chrome/(\d+[\.\d]+).+\(Chromium GOST\)'
+- regex: 'Chrome/(\d+\.[\.\d]+) .*\(Chromium GOST\)'
   name: 'Chromium GOST'
   version: '$1'
   engine:
@@ -349,7 +349,7 @@
     default: 'Gecko'
 
 # CoolBrowser (https://play.google.com/store/apps/details?id=com.easybrowser.browser.coolbrowser)
-- regex: 'Chrome/(\d+[\.\d]+).+AgentWeb.+UCBrowser'
+- regex: 'Chrome/(\d+\.[\.\d]+) .*AgentWeb.+UCBrowser'
   name: 'CoolBrowser'
   version: '$1'
   engine:
@@ -387,7 +387,7 @@
     default: 'Blink'
 
 # Qutebrowser (https://qutebrowser.org/)
-- regex: 'qutebrowser/(\d+[\.\d]+).+Chrome'
+- regex: 'qutebrowser/(\d+\.[\.\d]+) .*Chrome'
   name: 'Qutebrowser'
   version: '$1'
   engine:
@@ -791,6 +791,13 @@
   engine:
     default: 'Blink'
 
+# Wolvic VR Browser
+- regex: 'Wolvic/(\d+\.[.\d]+)'
+  name: 'Wolvic'
+  version: '$1'
+  engine:
+    default: 'Gecko'
+
 # Firefox Reality (https://mixedreality.mozilla.org/firefox-reality/)
 - regex: 'Mobile VR.+Firefox'
   name: 'Firefox Reality'
@@ -1168,7 +1175,7 @@
   version: '$1'
   engine:
     default: 'Blink'
-- regex: 'AlohaBrowser(?:/(\d+[\.\d]+))?'
+- regex: 'AlohaBrowser(?:App)?(?:/(\d+[\.\d]+))?'
   name: 'Aloha Browser'
   version: '$1'
 - regex: 'Aloha/'
@@ -1232,7 +1239,7 @@
   version: '$1'
 
 #Amigo
-- regex: 'Chrome/(\d+[\.\d]+).*MRCHROME'
+- regex: 'Chrome/(\d+\.[\.\d]+) .*MRCHROME'
   name: 'Amigo'
   version: '$1'
   engine:
@@ -1395,7 +1402,7 @@
     default: 'Gecko'
 
 #Swiftfox
-- regex: 'Firefox/(\d+[\.\d]+).*\(Swiftfox\)'
+- regex: 'Firefox/(\d+\.[\.\d]+) .*\(Swiftfox\)'
   name: 'Swiftfox'
   version: '$1'
   engine:
@@ -1581,7 +1588,7 @@
   version: '$1'
   engine:
     default: 'Blink'
-- regex: 'Opera/(\d+[\.\d]+).+Opera Mobi'
+- regex: 'Opera/(\d+\.[\.\d]+) .*Opera Mobi'
   name: 'Opera Mobile'
   version: '$1'
   engine:
@@ -1616,7 +1623,7 @@
     default: 'Presto'
     versions:
       15: 'Blink'
-- regex: '(?:Opera|OPR)[/ ](?:9.80.*Version/)?(\d+[\.\d]+).+Edition Next'
+- regex: '(?:Opera|OPR)[/ ](?:9.80.*Version/)?(\d+\.[\.\d]+) .*Edition Next'
   name: 'Opera Next'
   version: '$1'
   engine:
@@ -1699,7 +1706,17 @@
   name: 'Baidu Spark'
   version: '$1'
 
-# Yandex Browser Lite
+# Yandex Browser
+- regex: 'YaBrowser(?:/(\d+[\.\d]*)) YaApp_iOS'
+  name: 'Yandex Browser'
+  version: '$1'
+  engine:
+    default: 'WebKit'
+- regex: 'iP(?:hone|ad);.+YaBrowser(?:/(\d+[\.\d]*)) Mobile'
+  name: 'Yandex Browser'
+  version: '$1'
+  engine:
+    default: 'WebKit'
 - regex: 'YaBrowser(?:/(\d+[\.\d]*)) \(lite\)?'
   name: 'Yandex Browser Lite'
   version: '$1'
@@ -1789,7 +1806,7 @@
     default: 'Blink'
 
 # Iron
-- regex: 'Chrome(?:/(\d+[\.\d]+))?.*Iron'
+- regex: 'Chrome(?:/(\d+\.[\.\d]+) )?.*Iron'
   name: 'Iron'
   version: '$1'
   engine:
@@ -1869,7 +1886,7 @@
     versions:
       28: 'Blink'
 
-- regex: '(?:DDG-Android-|DuckDuckGo/)(\d+[\.\d]*)'
+- regex: '(?:DDG-Android-|DuckDuckGo/|ddg_android/)(\d+[\.\d]*)'
   name: 'DuckDuckGo Privacy Browser'
   version: '$1'
   engine:
@@ -1957,7 +1974,7 @@
     default: 'Blink'
 
 # Web Explorer
-- regex: 'Web Explorer/(\d+[\.\d]+).*Chrome'
+- regex: 'Web Explorer/(\d+\.[\.\d]+) .*Chrome'
   name: 'Web Explorer'
   version: '$1'
   engine:
@@ -2244,7 +2261,7 @@
 - regex: '(?:Tizen|SLP) ?Browser(?:/(\d+[\.\d]+))?'
   name: 'Tizen Browser'
   version: '$1'
-- regex: 'Tizen (?:\d+[\.\d]+).+ Version/(\d+[\.\d]+) (?:TV|Mobile|like)'
+- regex: 'Tizen (?:\d+\.[\.\d]+)[^\.\d].* Version/(\d+[\.\d]+) (?:TV|Mobile|like)'
   name: 'Tizen Browser'
   version: '$1'
   engine:
@@ -2413,7 +2430,7 @@
   version: '$1'
   engine:
     default: 'WebKit'
-- regex: 'GoogleEarth/(\d+[\.\d]+).+client:(?:Plus|Pro)'
+- regex: 'GoogleEarth/(\d+\.[\.\d]+)[^\.\d].*client:(?:Plus|Pro)'
   name: 'Google Earth Pro'
   version: '$1'
   engine:
@@ -2460,7 +2477,7 @@
   version: '$1'
   engine:
     default: 'Trident'
-- regex: 'MSIE (\d+[\.\d]+).*XBLWP7'
+- regex: 'MSIE (\d+\.[\.\d]+)[^\.\d].*XBLWP7'
   name: 'IE Mobile'
   version: '$1'
   engine:
@@ -2663,7 +2680,7 @@
   version: '$1'
   engine:
     default: 'WebKit'
-- regex: '(?:Version/(\d+[\.\d]+).*)?Mobile.*Safari/'
+- regex: '(?:Version/(\d+\.[\.\d]+) .*)?Mobile.*Safari/'
   name: 'Mobile Safari'
   version: '$1'
   engine:
@@ -2673,14 +2690,14 @@
   version: ''
   engine:
     default: 'WebKit'
-- regex: 'Version/(\d+[\.\d]+).*Safari/|(?:Safari|Safari(?:%20)?%E6%B5%8F%E8%A7%88%E5%99%A8)/?\d+'
+- regex: 'Version/(\d+\.[\.\d]+) .*Safari/|(?:Safari|Safari(?:%20)?%E6%B5%8F%E8%A7%88%E5%99%A8)/?\d+'
   name: 'Safari'
   version: '$1'
   engine:
     default: 'WebKit'
 
 # Dorado
-- regex: 'Dorado WAP-Browser(?:[/ ](\d+[\.\d]+))?'
+- regex: '(?:\w{1,5}[_ ])?Dorado WAP-Browser(?:[/ ](\d+[\.\d]+))?'
   name: 'Dorado'
   version: '$1'
 

data/regexes/client/hints/browsers.yml

@@ -86,6 +86,7 @@
 'com.hideitpro.vbrowser': 'vBrowser'
 'com.cgbrowser.rn': 'CG Browser'
 'com.azka.browser.anti.blokir': 'Azka Browser'
+'com.azka.browser': 'Azka Browser'
 'com.micromaxinfo.browser': 'Mmx Browser'
 'com.zeesitech.bitchutebrowser': 'Bitchute Browser'
 'nova.all.video.downloader': 'Nova Video Downloader Pro'
@@ -188,3 +189,7 @@
 'com.microsoft.emmx': 'Microsoft Edge'
 'com.explore.web.browser': 'Web Browser & Explorer'
 'privacy.explorer.fast.safe.browser': 'Privacy Explorer Fast Safe'
+'app.soundy.browser': 'Soundy Browser'
+'com.ivvi.browser': 'IVVI Browser'
+'com.nomone.vrbrowser': 'NOMone VR Browser'
+'com.opus.browser': 'Opus Browser'

data/regexes/client/libraries.yml

@@ -174,7 +174,7 @@
   name: 'HTTPie'
   version: '$1'
 
-- regex: 'rest-client/(\d+[\.\d]+).*ruby'
+- regex: 'rest-client/(\d+\.[\.\d]+) .*ruby'
   name: 'REST Client for Ruby'
   version: '$1'