browserx 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +39 -1
- data/README.md +122 -21
- data/bot_categories.yml +1522 -0
- data/bot_ip_ranges.yml +3035 -0
- data/bots.yml +1249 -0
- data/lib/browser/action_controller.rb +2 -1
- data/lib/browser/base.rb +20 -5
- data/lib/browser/bot/ip_ranges.rb +194 -0
- data/lib/browser/bot/known_bots_matcher.rb +1 -1
- data/lib/browser/bot.rb +98 -2
- data/lib/browser/browser.rb +69 -1
- data/lib/browser/cache.rb +92 -0
- data/lib/browser/client_hints.rb +73 -0
- data/lib/browser/cloudflare.rb +65 -0
- data/lib/browser/detection.rb +25 -0
- data/lib/browser/device/psvita.rb +1 -1
- data/lib/browser/device/surface.rb +1 -1
- data/lib/browser/device.rb +31 -7
- data/lib/browser/middleware/context.rb +2 -1
- data/lib/browser/platform.rb +34 -7
- data/lib/browser/version.rb +1 -1
- data/lib/browserx.rb +1 -1
- metadata +11 -4
data/lib/browser/base.rb
CHANGED
|
@@ -6,11 +6,14 @@ module Browser
|
|
|
6
6
|
|
|
7
7
|
attr_reader :ua
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
# headers: the Rack env (request.env), for the client hints and the
|
|
10
|
+
# Cloudflare headers.
|
|
11
|
+
def initialize(ua, accept_language: nil, headers: nil)
|
|
10
12
|
validate_size(:user_agent, ua.to_s)
|
|
11
13
|
|
|
12
14
|
@ua = ua
|
|
13
15
|
@accept_language_raw = accept_language.to_s
|
|
16
|
+
@headers = headers
|
|
14
17
|
end
|
|
15
18
|
|
|
16
19
|
# Return a meta info about this browser.
|
|
@@ -22,7 +25,7 @@ module Browser
|
|
|
22
25
|
def accept_language
|
|
23
26
|
@accept_language ||= begin
|
|
24
27
|
validate_size(:accept_language, @accept_language_raw)
|
|
25
|
-
|
|
28
|
+
Cache.accept_language(@accept_language_raw)
|
|
26
29
|
end
|
|
27
30
|
end
|
|
28
31
|
|
|
@@ -39,12 +42,20 @@ module Browser
|
|
|
39
42
|
|
|
40
43
|
# Return the platform.
|
|
41
44
|
def platform
|
|
42
|
-
@platform ||=
|
|
45
|
+
@platform ||= detection.platform.with_client_hints(client_hints)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def client_hints
|
|
49
|
+
@client_hints ||= @headers ? ClientHints.new(@headers) : ClientHints::NONE
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def cloudflare
|
|
53
|
+
@cloudflare ||= Cloudflare.new(@headers)
|
|
43
54
|
end
|
|
44
55
|
|
|
45
56
|
# Return the bot info.
|
|
46
57
|
def bot
|
|
47
|
-
@bot ||=
|
|
58
|
+
@bot ||= detection.bot
|
|
48
59
|
end
|
|
49
60
|
|
|
50
61
|
# Detect if current user agent is from a bot.
|
|
@@ -54,7 +65,7 @@ module Browser
|
|
|
54
65
|
|
|
55
66
|
# Return the device info.
|
|
56
67
|
def device
|
|
57
|
-
@device ||=
|
|
68
|
+
@device ||= detection.device.with_client_hints(client_hints, platform:)
|
|
58
69
|
end
|
|
59
70
|
|
|
60
71
|
# Detect if browser is Microsoft Internet Explorer.
|
|
@@ -273,6 +284,10 @@ module Browser
|
|
|
273
284
|
instance_of?(Epiphany) && detect_version?(full_version, expected_version)
|
|
274
285
|
end
|
|
275
286
|
|
|
287
|
+
private def detection
|
|
288
|
+
@detection ||= Cache.detection(ua)
|
|
289
|
+
end
|
|
290
|
+
|
|
276
291
|
private def validate_size(subject, input)
|
|
277
292
|
actual_bytesize = input.bytesize
|
|
278
293
|
size_limit =
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ipaddr"
|
|
4
|
+
|
|
5
|
+
module Browser
|
|
6
|
+
class Bot
|
|
7
|
+
# The IP ranges crawler owners publish, from bot_ip_ranges.yml (rewritten
|
|
8
|
+
# weekly by script/sync_ip_ranges.rb) or fetched again by `refresh`.
|
|
9
|
+
module IPRanges
|
|
10
|
+
GOOGLE_RANGES_URL = "https://developers.google.com/static/crawling/ipranges"
|
|
11
|
+
SOURCES = {
|
|
12
|
+
google: {
|
|
13
|
+
keywords: %w[
|
|
14
|
+
adsbot-google apis-google feedfetcher-google google-cloudvertexbot google-inspectiontool
|
|
15
|
+
google-read-aloud google-safety google-site-verification googleagent-urlcontext googlebot
|
|
16
|
+
googleother googleproducer mediapartners-google storebot-google
|
|
17
|
+
],
|
|
18
|
+
# https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot
|
|
19
|
+
domains: %w[googlebot.com google.com googleusercontent.com],
|
|
20
|
+
urls: %w[common-crawlers.json special-crawlers.json user-triggered-fetchers.json
|
|
21
|
+
user-triggered-fetchers-google.json user-triggered-agents.json].map { |file| "#{GOOGLE_RANGES_URL}/#{file}" }
|
|
22
|
+
},
|
|
23
|
+
bing: {
|
|
24
|
+
keywords: %w[adidxbot bingbot bingpreview microsoftpreview msnbot],
|
|
25
|
+
domains: %w[search.msn.com],
|
|
26
|
+
urls: %w[https://www.bing.com/toolbox/bingbot.json]
|
|
27
|
+
},
|
|
28
|
+
openai: {
|
|
29
|
+
keywords: %w[chatgpt-user gptbot oai-searchbot],
|
|
30
|
+
domains: [],
|
|
31
|
+
urls: %w[https://openai.com/gptbot.json https://openai.com/searchbot.json
|
|
32
|
+
https://openai.com/chatgpt-user.json]
|
|
33
|
+
},
|
|
34
|
+
apple: {
|
|
35
|
+
keywords: %w[applebot],
|
|
36
|
+
domains: %w[applebot.apple.com],
|
|
37
|
+
urls: %w[https://search.developer.apple.com/applebot.json]
|
|
38
|
+
},
|
|
39
|
+
perplexity: {
|
|
40
|
+
keywords: %w[perplexity-user perplexitybot],
|
|
41
|
+
domains: [],
|
|
42
|
+
urls: %w[https://www.perplexity.ai/perplexitybot.json https://www.perplexity.ai/perplexity-user.json]
|
|
43
|
+
},
|
|
44
|
+
duckduckgo: {
|
|
45
|
+
keywords: %w[duckassistbot duckduckbot],
|
|
46
|
+
domains: [],
|
|
47
|
+
urls: %w[https://duckduckgo.com/duckduckbot.json https://duckduckgo.com/duckassistbot.json]
|
|
48
|
+
}
|
|
49
|
+
}.freeze
|
|
50
|
+
FETCH_TIMEOUT_SECONDS = 10
|
|
51
|
+
# Owners move their files: Google's old URLs redirect to the new ones.
|
|
52
|
+
MAX_REDIRECTS = 3
|
|
53
|
+
DNS_TIMEOUT_SECONDS = 2
|
|
54
|
+
|
|
55
|
+
# The owner whose crawler the lowercase user agent claims to be.
|
|
56
|
+
def self.owner(ua)
|
|
57
|
+
SOURCES.each_key.find do |owner|
|
|
58
|
+
SOURCES[owner][:keywords].any? { |keyword| ua.include?(keyword) }
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def self.include?(owner, ip)
|
|
63
|
+
address = parse(ip)
|
|
64
|
+
return false unless address
|
|
65
|
+
|
|
66
|
+
starts, ends = ranges_by_owner.dig(owner, address.family)
|
|
67
|
+
return false unless starts
|
|
68
|
+
|
|
69
|
+
number = address.to_i
|
|
70
|
+
index = (starts.bsearch_index { |start| start > number } || starts.size) - 1
|
|
71
|
+
index >= 0 && number <= ends[index]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Google's documented method: the reverse lookup names one of the owner's
|
|
75
|
+
# domains, and that name resolves back to the IP. Two DNS queries.
|
|
76
|
+
def self.verified_by_dns?(owner, ip)
|
|
77
|
+
address = parse(ip)
|
|
78
|
+
domains = SOURCES.dig(owner, :domains)
|
|
79
|
+
return false unless address && domains&.any?
|
|
80
|
+
|
|
81
|
+
require "resolv"
|
|
82
|
+
Resolv::DNS.open do |dns|
|
|
83
|
+
dns.timeouts = DNS_TIMEOUT_SECONDS
|
|
84
|
+
host = dns.getname(address.to_s).to_s.downcase
|
|
85
|
+
domains.any? { |domain| host == domain || host.end_with?(".#{domain}") } &&
|
|
86
|
+
dns.getaddresses(host).any? { |resolved| IPAddr.new(resolved.to_s) == address }
|
|
87
|
+
end
|
|
88
|
+
rescue Resolv::ResolvError, Resolv::ResolvTimeout, SystemCallError
|
|
89
|
+
false
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Prefixes ("66.249.64.0/27"), keyed by owner.
|
|
93
|
+
def self.prefixes_by_owner
|
|
94
|
+
@prefixes_by_owner ||= Bot.load_yaml("bot_ip_ranges.yml").transform_keys(&:to_sym)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def self.ranges_by_owner
|
|
98
|
+
@ranges_by_owner ||= build_ranges(prefixes_by_owner)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# For long-lived processes: fetches every owner's ranges again and swaps
|
|
102
|
+
# them in at once. Raises Browser::Error and keeps the current ranges when
|
|
103
|
+
# a source fails.
|
|
104
|
+
def self.refresh
|
|
105
|
+
load(fetch)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def self.load(prefixes_by_owner)
|
|
109
|
+
ranges_by_owner = build_ranges(prefixes_by_owner)
|
|
110
|
+
@prefixes_by_owner = prefixes_by_owner
|
|
111
|
+
@ranges_by_owner = ranges_by_owner
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def self.fetch
|
|
115
|
+
SOURCES.to_h do |owner, source|
|
|
116
|
+
addresses = source[:urls].flat_map { |url| fetch_prefixes(url) }.map do |prefix|
|
|
117
|
+
IPAddr.new(prefix)
|
|
118
|
+
rescue ArgumentError
|
|
119
|
+
raise Error, "#{owner}: invalid prefix #{prefix.inspect}"
|
|
120
|
+
end
|
|
121
|
+
sorted = addresses.uniq.sort_by { |address| [address.family, address.to_i, address.prefix] }
|
|
122
|
+
[owner, sorted.map { |address| "#{address}/#{address.prefix}" }]
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def self.fetch_prefixes(url)
|
|
127
|
+
require "json"
|
|
128
|
+
require "net/http"
|
|
129
|
+
uri = URI(url)
|
|
130
|
+
response = nil
|
|
131
|
+
(MAX_REDIRECTS + 1).times do
|
|
132
|
+
response = Net::HTTP.start(uri.host, uri.port, use_ssl: true,
|
|
133
|
+
open_timeout: FETCH_TIMEOUT_SECONDS, read_timeout: FETCH_TIMEOUT_SECONDS) do |http|
|
|
134
|
+
http.get(uri.request_uri, "User-Agent" => "BrowserX/#{VERSION} (+https://github.com/next-gen-gems/browserx)")
|
|
135
|
+
end
|
|
136
|
+
break unless response.is_a?(Net::HTTPRedirection)
|
|
137
|
+
|
|
138
|
+
uri = uri.merge(response["Location"])
|
|
139
|
+
end
|
|
140
|
+
raise Error, "#{url}: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
|
|
141
|
+
|
|
142
|
+
prefixes = JSON.parse(response.body).fetch("prefixes").map do |prefix|
|
|
143
|
+
prefix["ipv4Prefix"] || prefix["ipv6Prefix"]
|
|
144
|
+
end
|
|
145
|
+
raise Error, "#{url}: no prefixes" if prefixes.empty?
|
|
146
|
+
|
|
147
|
+
prefixes
|
|
148
|
+
rescue JSON::ParserError, KeyError, Timeout::Error, IOError, SystemCallError, OpenSSL::SSL::SSLError => exception
|
|
149
|
+
raise Error, "#{url}: #{exception.message}"
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Per owner and address family, the merged ranges as two sorted arrays of
|
|
153
|
+
# integers, first and last addresses, for a binary search.
|
|
154
|
+
def self.build_ranges(prefixes_by_owner)
|
|
155
|
+
prefixes_by_owner.to_h do |owner, prefixes|
|
|
156
|
+
ranges_by_family = prefixes.map { |prefix| IPAddr.new(prefix) }.group_by(&:family)
|
|
157
|
+
[owner.to_sym, ranges_by_family.transform_values { |addresses| merge(addresses) }]
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def self.merge(addresses)
|
|
162
|
+
ranges = addresses.map do |address|
|
|
163
|
+
range = address.to_range
|
|
164
|
+
[range.begin.to_i, range.end.to_i]
|
|
165
|
+
end
|
|
166
|
+
ranges.sort!
|
|
167
|
+
starts = []
|
|
168
|
+
ends = []
|
|
169
|
+
ranges.each do |first, last|
|
|
170
|
+
if ends.any? && first <= ends.last + 1
|
|
171
|
+
ends[-1] = last if last > ends.last
|
|
172
|
+
else
|
|
173
|
+
starts << first
|
|
174
|
+
ends << last
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
[starts, ends]
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# A forged or malformed X-Forwarded-For must not raise in a request.
|
|
181
|
+
def self.parse(ip)
|
|
182
|
+
ip = ip.to_s
|
|
183
|
+
return if ip.include?("/")
|
|
184
|
+
|
|
185
|
+
address = IPAddr.new(ip)
|
|
186
|
+
address.ipv4_mapped? ? address.native : address
|
|
187
|
+
rescue ArgumentError
|
|
188
|
+
nil
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
private_class_method :fetch_prefixes, :build_ranges, :merge, :parse
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
end
|
data/lib/browser/bot.rb
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
module Browser
|
|
4
4
|
class Bot
|
|
5
5
|
GENERIC_NAME = "Generic Bot"
|
|
6
|
+
AI_CATEGORIES = %i[ai_agent ai_assistant ai_crawler ai_search].freeze
|
|
7
|
+
USER_TRIGGERED_CATEGORIES = %i[ai_agent ai_assistant].freeze
|
|
6
8
|
|
|
7
9
|
def self.matchers
|
|
8
10
|
@matchers ||= default_matchers
|
|
@@ -28,6 +30,13 @@ module Browser
|
|
|
28
30
|
@search_engines ||= load_yaml("search_engines.yml")
|
|
29
31
|
end
|
|
30
32
|
|
|
33
|
+
def self.category_by_keyword
|
|
34
|
+
@category_by_keyword ||=
|
|
35
|
+
load_yaml("bot_categories.yml").each_with_object({}) do |(category, keywords), category_by_keyword|
|
|
36
|
+
keywords.each { |keyword| category_by_keyword[keyword] = category.to_sym }
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
31
40
|
# One regexp per keyword list (an Array, or a Hash keyed by keyword),
|
|
32
41
|
# rebuilt when the list grows or is replaced.
|
|
33
42
|
def self.keywords_pattern(keywords)
|
|
@@ -41,6 +50,43 @@ module Browser
|
|
|
41
50
|
pattern
|
|
42
51
|
end
|
|
43
52
|
|
|
53
|
+
# bots.yml keywords nested by byte, each end holding the keyword's rank.
|
|
54
|
+
# Walked from each byte of a user agent, it follows the few branches that
|
|
55
|
+
# byte opens, where a regexp of a thousand keywords tries most of them.
|
|
56
|
+
def self.known_bots_trie
|
|
57
|
+
list, size, trie = @known_bots_trie
|
|
58
|
+
return trie if list.equal?(bots) && size == bots.size
|
|
59
|
+
|
|
60
|
+
trie = {keywords: bots.keys}
|
|
61
|
+
trie[:keywords].each_with_index do |keyword, rank|
|
|
62
|
+
leaf = keyword.to_s.each_byte.reduce(trie) { |node, byte| node[byte] ||= {} }
|
|
63
|
+
leaf[:rank] ||= rank
|
|
64
|
+
end
|
|
65
|
+
@known_bots_trie = [bots, bots.size, trie]
|
|
66
|
+
trie
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# The rank of a bots.yml keyword the user agent holds: the lowest, or the
|
|
70
|
+
# first one found when `lowest: false`.
|
|
71
|
+
def self.known_bot_rank(ua, lowest: true)
|
|
72
|
+
trie = known_bots_trie
|
|
73
|
+
found = nil
|
|
74
|
+
ua.bytesize.times do |start|
|
|
75
|
+
node = trie
|
|
76
|
+
position = start
|
|
77
|
+
while (node = node[ua.getbyte(position)])
|
|
78
|
+
rank = node[:rank]
|
|
79
|
+
if rank
|
|
80
|
+
return rank unless lowest
|
|
81
|
+
|
|
82
|
+
found = rank if found.nil? || rank < found
|
|
83
|
+
end
|
|
84
|
+
position += 1
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
found
|
|
88
|
+
end
|
|
89
|
+
|
|
44
90
|
# A Regexp.union of hundreds of words tries each of them at every
|
|
45
91
|
# position; nested by common prefix, the regexp follows a single branch
|
|
46
92
|
# per character.
|
|
@@ -76,8 +122,14 @@ module Browser
|
|
|
76
122
|
@ua = ua.downcase.strip
|
|
77
123
|
end
|
|
78
124
|
|
|
125
|
+
# A bot is shared by the browsers of its user agent: each answer holds
|
|
126
|
+
# until a matcher list or the bot list changes.
|
|
79
127
|
def bot?
|
|
80
|
-
|
|
128
|
+
generation = Cache.generation
|
|
129
|
+
unless @bot&.first == generation
|
|
130
|
+
@bot = [generation, !bot_exception? && detect_bot?]
|
|
131
|
+
end
|
|
132
|
+
@bot.last
|
|
81
133
|
end
|
|
82
134
|
|
|
83
135
|
def why?
|
|
@@ -91,7 +143,51 @@ module Browser
|
|
|
91
143
|
def name
|
|
92
144
|
return unless bot?
|
|
93
145
|
|
|
94
|
-
|
|
146
|
+
generation = Cache.generation
|
|
147
|
+
unless @name&.first == generation
|
|
148
|
+
@name = [generation, self.class.bots[known_bot_keyword] || GENERIC_NAME]
|
|
149
|
+
end
|
|
150
|
+
@name.last
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Test drivers have a category, but no bot? (bot_categories.yml).
|
|
154
|
+
def category
|
|
155
|
+
return if bot_exception?
|
|
156
|
+
|
|
157
|
+
keyword = bot? && known_bot_keyword
|
|
158
|
+
keyword ||= ua[self.class.keywords_pattern(self.class.category_by_keyword)]
|
|
159
|
+
self.class.category_by_keyword[keyword]
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def ai?
|
|
163
|
+
AI_CATEGORIES.include?(category)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def user_triggered?
|
|
167
|
+
USER_TRIGGERED_CATEGORIES.include?(category)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# The owner publishing the IP ranges of the crawler this user agent claims
|
|
171
|
+
# to be (:google, :bing, :openai, :apple, :perplexity, :duckduckgo).
|
|
172
|
+
def owner
|
|
173
|
+
return unless bot?
|
|
174
|
+
|
|
175
|
+
@owner = IPRanges.owner(ua) unless defined?(@owner)
|
|
176
|
+
@owner
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# Whether the IP belongs to the owner's published ranges. reverse_dns: true
|
|
180
|
+
# also asks the DNS for an IP outside them (Google, Bing and Apple): two
|
|
181
|
+
# network queries.
|
|
182
|
+
def verified?(ip, reverse_dns: false)
|
|
183
|
+
return false unless owner
|
|
184
|
+
|
|
185
|
+
IPRanges.include?(owner, ip) || (reverse_dns && IPRanges.verified_by_dns?(owner, ip))
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
private def known_bot_keyword
|
|
189
|
+
rank = self.class.known_bot_rank(ua)
|
|
190
|
+
self.class.known_bots_trie[:keywords][rank] if rank
|
|
95
191
|
end
|
|
96
192
|
|
|
97
193
|
private def bot_exception?
|
data/lib/browser/browser.rb
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
if Gem.loaded_specs.key?("browser")
|
|
4
|
+
raise LoadError, "browserx replaces the browser gem, they can't be loaded together: " \
|
|
5
|
+
"remove gem \"browser\" from the Gemfile (or whatever depends on it)"
|
|
6
|
+
end
|
|
7
|
+
|
|
3
8
|
require "yaml"
|
|
4
9
|
require "pathname"
|
|
5
10
|
|
|
@@ -42,11 +47,16 @@ require_relative "bot"
|
|
|
42
47
|
require_relative "bot/empty_user_agent_matcher"
|
|
43
48
|
require_relative "bot/keyword_matcher"
|
|
44
49
|
require_relative "bot/known_bots_matcher"
|
|
50
|
+
require_relative "bot/ip_ranges"
|
|
45
51
|
|
|
46
52
|
require_relative "middleware"
|
|
47
53
|
require_relative "platform"
|
|
48
54
|
require_relative "device"
|
|
49
55
|
require_relative "meta"
|
|
56
|
+
require_relative "client_hints"
|
|
57
|
+
require_relative "cloudflare"
|
|
58
|
+
require_relative "detection"
|
|
59
|
+
require_relative "cache"
|
|
50
60
|
|
|
51
61
|
module Browser
|
|
52
62
|
EMPTY_STRING = ""
|
|
@@ -63,11 +73,23 @@ module Browser
|
|
|
63
73
|
|
|
64
74
|
class << self
|
|
65
75
|
attr_accessor :user_agent_size_limit, :accept_language_size_limit
|
|
76
|
+
attr_reader :cache_size
|
|
66
77
|
end
|
|
67
78
|
|
|
68
79
|
self.user_agent_size_limit = 2048
|
|
69
80
|
self.accept_language_size_limit = 2048
|
|
70
81
|
|
|
82
|
+
def self.cache_size=(size)
|
|
83
|
+
unless size.is_a?(Integer) && size >= 0
|
|
84
|
+
raise ArgumentError, "cache_size must be an Integer >= 0, got #{size.inspect}"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
@cache_size = size
|
|
88
|
+
Cache.clear
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
self.cache_size = 1000
|
|
92
|
+
|
|
71
93
|
# Hold the list of browser matchers.
|
|
72
94
|
# Order is important.
|
|
73
95
|
def self.matchers
|
|
@@ -107,10 +129,56 @@ module Browser
|
|
|
107
129
|
]
|
|
108
130
|
end
|
|
109
131
|
|
|
132
|
+
# A pattern that every user agent a built-in browser matches also matches:
|
|
133
|
+
# Browser.new skips, without building it, a browser whose pattern fails.
|
|
134
|
+
REQUIRED_PATTERN_BY_MATCHER = {
|
|
135
|
+
Nokia => /S40OviBrowser/,
|
|
136
|
+
UCBrowser => /UCBrowser/,
|
|
137
|
+
PhantomJS => /PhantomJS/,
|
|
138
|
+
BlackBerry => /BlackBerry|BB10/,
|
|
139
|
+
Opera => %r{Opera|OPR/},
|
|
140
|
+
Edge => %r{Edg|Trident/8},
|
|
141
|
+
InternetExplorer => %r{MSIE|Trident/},
|
|
142
|
+
Firefox => /Firefox|FxiOS/,
|
|
143
|
+
Otter => /Otter/,
|
|
144
|
+
Epiphany => /Epiphany/,
|
|
145
|
+
Facebook => /FBAV|FBAN/,
|
|
146
|
+
Instagram => /Instagram/,
|
|
147
|
+
Snapchat => /Snapchat/,
|
|
148
|
+
Weibo => /__weibo__/i,
|
|
149
|
+
MicroMessenger => /MicroMessenger/i,
|
|
150
|
+
QQ => %r{QQ/|QQBrowser}i,
|
|
151
|
+
Alipay => /AlipayClient/i,
|
|
152
|
+
Electron => /Electron/,
|
|
153
|
+
Yandex => /YaBrowser/,
|
|
154
|
+
Sputnik => /SputnikBrowser/,
|
|
155
|
+
DuckDuckGo => /DuckDuckGo/,
|
|
156
|
+
SamsungBrowser => /SamsungBrowser/,
|
|
157
|
+
HuaweiBrowser => /HuaweiBrowser/i,
|
|
158
|
+
MiuiBrowser => /MiuiBrowser/,
|
|
159
|
+
Maxthon => /Maxthon/i,
|
|
160
|
+
SougouBrowser => /(?i:SogouMobileBrowser)|\bSE\b/,
|
|
161
|
+
GoogleSearchApp => /GSA/
|
|
162
|
+
}.compare_by_identity.freeze
|
|
163
|
+
|
|
110
164
|
def self.new(user_agent, **)
|
|
165
|
+
user_agent ||= EMPTY_STRING
|
|
166
|
+
browser_class = Cache.cached_detection(user_agent)&.browser_class
|
|
167
|
+
return browser_class.new(user_agent, **) if browser_class
|
|
168
|
+
|
|
169
|
+
browser = detect(user_agent, **)
|
|
170
|
+
Cache.detection(user_agent).browser_class = browser.class
|
|
171
|
+
browser
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def self.detect(user_agent, **)
|
|
111
175
|
matchers.each do |matcher_class|
|
|
112
|
-
|
|
176
|
+
required_pattern = REQUIRED_PATTERN_BY_MATCHER[matcher_class]
|
|
177
|
+
next if required_pattern && !required_pattern.match?(user_agent)
|
|
178
|
+
|
|
179
|
+
matcher = matcher_class.new(user_agent, **)
|
|
113
180
|
return matcher if matcher.match?
|
|
114
181
|
end
|
|
115
182
|
end
|
|
183
|
+
private_class_method :detect
|
|
116
184
|
end
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Browser
|
|
4
|
+
# The detections of the last user agents and the parsed Accept-Language
|
|
5
|
+
# headers, shared by the threads of a process. Reads take no lock: under the
|
|
6
|
+
# GVL a Hash read is atomic, and a thread that misses detects again. Writes
|
|
7
|
+
# do, for the Rubies without a GVL.
|
|
8
|
+
module Cache
|
|
9
|
+
@mutex = Mutex.new
|
|
10
|
+
@detection_by_user_agent = {}
|
|
11
|
+
@languages_by_header = {}
|
|
12
|
+
@generation = 0
|
|
13
|
+
|
|
14
|
+
def self.cached_detection(ua)
|
|
15
|
+
current_detection_by_user_agent[ua]
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.detection(ua)
|
|
19
|
+
return Detection.new(ua) if Browser.cache_size.zero?
|
|
20
|
+
|
|
21
|
+
cached_detection(ua) || @mutex.synchronize do
|
|
22
|
+
ua = -ua
|
|
23
|
+
store(@detection_by_user_agent, ua, Detection.new(ua))
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# A copy each time: callers may change the array they get.
|
|
28
|
+
def self.accept_language(header)
|
|
29
|
+
return AcceptLanguage.parse(header) if Browser.cache_size.zero?
|
|
30
|
+
|
|
31
|
+
languages = @languages_by_header[header] || @mutex.synchronize do
|
|
32
|
+
store(@languages_by_header, -header,
|
|
33
|
+
AcceptLanguage.parse(header).freeze)
|
|
34
|
+
end
|
|
35
|
+
languages.dup
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Bumped when a matcher list, the bot list or its exceptions change: the
|
|
39
|
+
# matchers are public arrays that apps extend at runtime.
|
|
40
|
+
def self.generation
|
|
41
|
+
return @generation if lists_unchanged?
|
|
42
|
+
|
|
43
|
+
@mutex.synchronize do
|
|
44
|
+
@generation += 1
|
|
45
|
+
@detection_by_user_agent.clear
|
|
46
|
+
remember_lists
|
|
47
|
+
end
|
|
48
|
+
@generation
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def self.clear
|
|
52
|
+
@mutex.synchronize do
|
|
53
|
+
@detection_by_user_agent.clear
|
|
54
|
+
@languages_by_header.clear
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The bot list holds hundreds of entries: compared by identity and size,
|
|
59
|
+
# like Bot.keywords_pattern, not entry by entry on every request.
|
|
60
|
+
def self.lists_unchanged?
|
|
61
|
+
@browser_matchers == Browser.matchers &&
|
|
62
|
+
@platform_matchers == Platform.matchers &&
|
|
63
|
+
@device_matchers == Device.matchers &&
|
|
64
|
+
@bot_matchers == Bot.matchers &&
|
|
65
|
+
@bot_exceptions == Bot.bot_exceptions &&
|
|
66
|
+
@bots.equal?(Bot.bots) && @bots_size == Bot.bots.size
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def self.remember_lists
|
|
70
|
+
@browser_matchers = Browser.matchers.dup
|
|
71
|
+
@platform_matchers = Platform.matchers.dup
|
|
72
|
+
@device_matchers = Device.matchers.dup
|
|
73
|
+
@bot_matchers = Bot.matchers.dup
|
|
74
|
+
@bot_exceptions = Bot.bot_exceptions.dup
|
|
75
|
+
@bots = Bot.bots
|
|
76
|
+
@bots_size = Bot.bots.size
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def self.current_detection_by_user_agent
|
|
80
|
+
generation
|
|
81
|
+
@detection_by_user_agent
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def self.store(value_by_key, key, value)
|
|
85
|
+
value_by_key.clear if value_by_key.size >= Browser.cache_size
|
|
86
|
+
value_by_key[key] = value
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private_class_method :lists_unchanged?, :remember_lists,
|
|
90
|
+
:current_detection_by_user_agent, :store
|
|
91
|
+
end
|
|
92
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Browser
|
|
4
|
+
# The User-Agent Client Hints of Chromium browsers, read from a Rack env.
|
|
5
|
+
# Safari and Firefox send none: every answer is then nil.
|
|
6
|
+
class ClientHints
|
|
7
|
+
# A browser's hints are a few dozen bytes: longer values are ignored.
|
|
8
|
+
HEADER_SIZE_LIMIT_BYTES = 512
|
|
9
|
+
PLATFORM_CLASS_BY_NAME = {
|
|
10
|
+
"Android" => Platform::Android,
|
|
11
|
+
"Chrome OS" => Platform::ChromeOS,
|
|
12
|
+
"Linux" => Platform::Linux,
|
|
13
|
+
"macOS" => Platform::Mac,
|
|
14
|
+
"Windows" => Platform::Windows
|
|
15
|
+
}.freeze
|
|
16
|
+
# https://learn.microsoft.com/en-us/microsoft-edge/web-platform/how-to-detect-win11
|
|
17
|
+
WINDOWS_11_PLATFORM_VERSION = 13
|
|
18
|
+
BRAND_PATTERN = /"([^"\\]*)"\s*;\s*v\s*=\s*"([^"\\]*)"/
|
|
19
|
+
# GREASE brands, sent so that servers don't depend on the list's order.
|
|
20
|
+
GREASE_BRAND_PATTERN = /\ANot.A.Brand\z/i
|
|
21
|
+
QUOTED_STRING_PATTERN = /\A\s*"([^"\\]+)"\s*\z/
|
|
22
|
+
MOBILE_BY_VALUE = {"?1" => true, "?0" => false}.freeze
|
|
23
|
+
|
|
24
|
+
# Brand versions keyed by brand ("Google Chrome" => "142"), from Sec-CH-UA.
|
|
25
|
+
attr_reader :version_by_brand
|
|
26
|
+
# Sec-CH-UA-Platform ("Windows", "macOS", "Android"…).
|
|
27
|
+
attr_reader :platform
|
|
28
|
+
# Sec-CH-UA-Platform-Version ("15.0.0"), sent only to a site asking for it
|
|
29
|
+
# with an Accept-CH: Sec-CH-UA-Platform-Version response header.
|
|
30
|
+
attr_reader :platform_version
|
|
31
|
+
|
|
32
|
+
def initialize(headers)
|
|
33
|
+
@version_by_brand = parse_brands(header(headers, "HTTP_SEC_CH_UA"))
|
|
34
|
+
@mobile = MOBILE_BY_VALUE[header(headers, "HTTP_SEC_CH_UA_MOBILE")&.strip]
|
|
35
|
+
@platform = quoted_string(header(headers, "HTTP_SEC_CH_UA_PLATFORM"))
|
|
36
|
+
@platform_version = quoted_string(header(headers, "HTTP_SEC_CH_UA_PLATFORM_VERSION"))
|
|
37
|
+
freeze
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Sec-CH-UA-Mobile: nil when the browser did not send it.
|
|
41
|
+
def mobile?
|
|
42
|
+
@mobile
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def platform_class
|
|
46
|
+
PLATFORM_CLASS_BY_NAME[platform]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Windows 11 says "Windows NT 10.0" in its user agent: only its platform
|
|
50
|
+
# version tells it from Windows 10.
|
|
51
|
+
def windows11?
|
|
52
|
+
platform == "Windows" && platform_version.to_i >= WINDOWS_11_PLATFORM_VERSION
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private def header(headers, name)
|
|
56
|
+
value = headers[name]
|
|
57
|
+
value if value.is_a?(String) && value.bytesize <= HEADER_SIZE_LIMIT_BYTES
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private def parse_brands(value)
|
|
61
|
+
return EMPTY_HASH unless value
|
|
62
|
+
|
|
63
|
+
value.scan(BRAND_PATTERN).to_h.reject { |brand, _version| GREASE_BRAND_PATTERN.match?(brand) }.freeze
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private def quoted_string(value)
|
|
67
|
+
value&.[](QUOTED_STRING_PATTERN, 1)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
EMPTY_HASH = {}.freeze
|
|
71
|
+
NONE = new(EMPTY_HASH)
|
|
72
|
+
end
|
|
73
|
+
end
|