rails_mind 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -1
- data/CONTRIBUTING.md +6 -7
- data/README.md +11 -6
- data/docs/compatibility.md +31 -14
- data/docs/sdk.md +141 -39
- data/lib/generators/rails_mind/install/install_generator.rb +0 -12
- data/lib/generators/rails_mind/install/templates/initializer.rb.tt +8 -1
- data/lib/rails_mind/client.rb +3 -2
- data/lib/rails_mind/configuration.rb +8 -2
- data/lib/rails_mind/context.rb +8 -2
- data/lib/rails_mind/diagnostic.rb +18 -3
- data/lib/rails_mind/integrations/rails.rb +76 -9
- data/lib/rails_mind/page_view.rb +176 -0
- data/lib/rails_mind/redactor.rb +127 -12
- data/lib/rails_mind/transport.rb +14 -0
- data/lib/rails_mind/version.rb +1 -1
- data/lib/rails_mind/visitor.rb +134 -0
- data/lib/tasks/rails_mind.rake +8 -2
- metadata +5 -5
- data/javascript/rails_mind.js +0 -49
- data/lib/rails_mind/integrations/ahoy.rb +0 -68
data/lib/rails_mind/context.rb
CHANGED
|
@@ -39,8 +39,14 @@ module RailsMind
|
|
|
39
39
|
Thread.current[:rails_mind_context] = previous
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
UNSET = Object.new.freeze
|
|
43
|
+
|
|
44
|
+
# Signing in names the user and account; the visitor stays the one this
|
|
45
|
+
# request already carries unless visitor_id is passed (nil clears it).
|
|
46
|
+
def identify(user_id: nil, account_id: nil, visitor_id: UNSET)
|
|
47
|
+
values = { user_id: user_id&.to_s, account_id: account_id&.to_s }
|
|
48
|
+
values[:visitor_id] = visitor_id&.to_s unless UNSET.equal?(visitor_id)
|
|
49
|
+
current.merge!(values)
|
|
44
50
|
end
|
|
45
51
|
|
|
46
52
|
def suppressed? = Thread.current[:rails_mind_suppressed] == true
|
|
@@ -22,19 +22,34 @@ module RailsMind
|
|
|
22
22
|
release_present: !@config.release.to_s.strip.empty?,
|
|
23
23
|
enabled: @config.enabled?,
|
|
24
24
|
automatic_sampling: @config.automatic_sampling,
|
|
25
|
+
page_views: @config.page_views,
|
|
26
|
+
visitor_cookie: @config.visitor_cookie,
|
|
27
|
+
link_page_views_to_users: @config.link_page_views_to_users,
|
|
25
28
|
plan: [
|
|
26
29
|
"Add one initializer. No database migrations or Gemfile changes.",
|
|
27
30
|
"Requests: automatic sampling of routine successes; jobs via Rails notifications and errors via Rails.error stay unsampled.",
|
|
28
|
-
|
|
31
|
+
"Page views: HTML pages your controllers render are counted with a first-party visitor cookie; config.page_views, config.visitor_cookie and config.link_page_views_to_users turn parts off.",
|
|
32
|
+
("Ahoy detected: RailsMind no longer reads Ahoy and counts page views itself. Remove any require \"rails_mind/integrations/ahoy\" and RailsMind::Integrations.mirror_ahoy! call, or the app fails at boot; Ahoy keeps its own tables." if detected["ahoy_matey"]),
|
|
29
33
|
detected["opentelemetry-sdk"] ? "OpenTelemetry detected: attach RailsMind exporter after existing SDK setup; preserve sampler and exporters." : "APM traces: optionally add selected OpenTelemetry libraries and explicitly configure one provider.",
|
|
30
34
|
detected["flipper"] ? "Flipper detected: observe notifications only; preserve adapter, groups, targeting and cache." : "Flags: Flipper is optional. No flag configuration is created.",
|
|
31
35
|
"Existing error reporters stay installed. RailsMind deduplicates its own collector only.",
|
|
32
|
-
"Browser: optional Ahoy adapter; enable only after checking existing pageview handlers.",
|
|
33
36
|
"Business/error signals are not sampled; routine requests follow the hosted policy. Local opt-out does not disable server sampling or quotas. Inspect RailsMind.stats for queue losses and unconfirmed delivery."
|
|
34
|
-
],
|
|
37
|
+
].compact,
|
|
35
38
|
limitations: [ "Compatibility applies to tested versions only; this is not a historical import.",
|
|
36
39
|
"No network check is performed and credential values are never printed." ]
|
|
37
40
|
}
|
|
38
41
|
end
|
|
42
|
+
|
|
43
|
+
# What keeps collection off, one sentence each, naming the setting to change.
|
|
44
|
+
def problems
|
|
45
|
+
[].tap do |found|
|
|
46
|
+
if @config.token.to_s.strip.empty?
|
|
47
|
+
found << "RAILS_MIND_KEY is not set, so collection is off. Set it in this app's environment to the ingest key from RailsMind Setup, restart, then run bin/rails rails_mind:verify."
|
|
48
|
+
end
|
|
49
|
+
if @config.endpoint.to_s.strip.empty?
|
|
50
|
+
found << "The RailsMind endpoint is blank, so collection is off. Leave config.endpoint and RAILS_MIND_ENDPOINT unset to use #{Configuration::DEFAULT_ENDPOINT}, or set your RailsMind instance's URL."
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
39
54
|
end
|
|
40
55
|
end
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
require "active_support/notifications"
|
|
2
2
|
require "active_support/inflector"
|
|
3
3
|
require "digest"
|
|
4
|
+
require_relative "../visitor"
|
|
5
|
+
require_relative "../page_view"
|
|
4
6
|
|
|
5
7
|
module RailsMind
|
|
6
8
|
module Integrations
|
|
@@ -44,7 +46,10 @@ module RailsMind
|
|
|
44
46
|
Context.with_trace do
|
|
45
47
|
previous = Thread.current[:rails_mind_queries]
|
|
46
48
|
Thread.current[:rails_mind_queries] = { count: 0, fingerprints: Hash.new(0), shapes: {} }
|
|
47
|
-
|
|
49
|
+
Visitor.start(env)
|
|
50
|
+
response = @app.call(env)
|
|
51
|
+
Visitor.finish(env, response)
|
|
52
|
+
response
|
|
48
53
|
rescue StandardError => error
|
|
49
54
|
# Capture identity before this request boundary unwinds. Object
|
|
50
55
|
# dedup still leaves other error reporters unaffected.
|
|
@@ -113,18 +118,72 @@ module RailsMind
|
|
|
113
118
|
# was cut), numbers and hex literals become ?, and comments go. Double
|
|
114
119
|
# quotes are identifiers, except on MySQL, where they can be strings.
|
|
115
120
|
LEXEME = /(?<!\w)[Ee]'(?:[^'\\]|\\.|'')*'?|'(?:[^'\\]|\\.|'')*'?|\$(\w*)\$.*?(?:\$\1\$|\z)|"(?:[^"\\]|\\.|"")*"?|`[^`]*`?|--[^\n]*|\/\*.*?(?:\*\/|\z)|\b0[xX]\h+\b|\b\d+(?:\.\d+)?(?:[eE][-+]?\d+)?\b/m
|
|
121
|
+
# The same lexemes where a backslash escapes nothing outside E'...', as on
|
|
122
|
+
# PostgreSQL (Rails turns standard_conforming_strings on) and SQLite. Which
|
|
123
|
+
# rule applies is not always known (MySQL can turn backslash escapes off),
|
|
124
|
+
# so a statement with a backslash is read both ways and anything either
|
|
125
|
+
# reading takes for a literal goes. Without a backslash both agree.
|
|
126
|
+
STANDARD_LEXEME = /(?<!\w)[Ee]'(?:[^'\\]|\\.|'')*'?|'(?:[^']|'')*'?|\$(\w*)\$.*?(?:\$\1\$|\z)|"(?:[^"]|"")*"?|`[^`]*`?|--[^\n]*|\/\*.*?(?:\*\/|\z)|\b0[xX]\h+\b|\b\d+(?:\.\d+)?(?:[eE][-+]?\d+)?\b/m
|
|
116
127
|
|
|
117
128
|
def self.normalize(sql, mysql: false)
|
|
118
|
-
sql.scrub("?")
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
when "`" then token
|
|
122
|
-
when '"' then mysql ? "?" : token
|
|
123
|
-
else "?"
|
|
124
|
-
end
|
|
125
|
-
end.gsub(/\s+/, " ").strip
|
|
129
|
+
sql = sql.scrub("?")
|
|
130
|
+
text = sql.include?("\\") ? read_both_ways(sql, mysql) : sql.gsub(LEXEME) { |token| replacement(token, mysql) }
|
|
131
|
+
text.gsub(/\s+/, " ").strip
|
|
126
132
|
end
|
|
127
133
|
|
|
134
|
+
def self.replacement(token, mysql)
|
|
135
|
+
case token[0]
|
|
136
|
+
when "-", "/" then " "
|
|
137
|
+
when "`" then token
|
|
138
|
+
when '"' then mysql ? "?" : token
|
|
139
|
+
else "?"
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Literal spans from either reading merge into one ?; a comment that
|
|
144
|
+
# overlaps one joins it. Kept identifiers stay only where no reading saw a
|
|
145
|
+
# literal.
|
|
146
|
+
def self.read_both_ways(sql, mysql)
|
|
147
|
+
literals, comments = [], []
|
|
148
|
+
[ STANDARD_LEXEME, LEXEME ].each do |pattern|
|
|
149
|
+
sql.scan(pattern) do
|
|
150
|
+
token = Regexp.last_match
|
|
151
|
+
case replacement(token[0], mysql)
|
|
152
|
+
when "?" then literals << token.offset(0)
|
|
153
|
+
when " " then comments << token.offset(0)
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
literals = merge(literals)
|
|
158
|
+
loop do
|
|
159
|
+
# Merged literal spans are sorted and disjoint, so their ends are too.
|
|
160
|
+
joining, comments = comments.partition do |start, finish|
|
|
161
|
+
overlap = literals.bsearch { |_, last| last > start }
|
|
162
|
+
overlap && overlap[0] < finish
|
|
163
|
+
end
|
|
164
|
+
break if joining.empty?
|
|
165
|
+
literals = merge(literals + joining)
|
|
166
|
+
end
|
|
167
|
+
spans = (literals.map { |span| span + [ "?" ] } + merge(comments).map { |span| span + [ " " ] }).sort
|
|
168
|
+
output, position = +"", 0
|
|
169
|
+
spans.each do |start, finish, mark|
|
|
170
|
+
output << sql[position...start] << mark
|
|
171
|
+
position = finish
|
|
172
|
+
end
|
|
173
|
+
output << sql[position..]
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def self.merge(spans)
|
|
177
|
+
spans.sort.each_with_object([]) do |(start, finish), merged|
|
|
178
|
+
if merged.any? && start < merged.last[1]
|
|
179
|
+
merged.last[1] = [ merged.last[1], finish ].max
|
|
180
|
+
else
|
|
181
|
+
merged << [ start, finish ]
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
private_class_method :replacement, :read_both_ways, :merge
|
|
186
|
+
|
|
128
187
|
class << self
|
|
129
188
|
def install!
|
|
130
189
|
return if @subscriptions
|
|
@@ -147,6 +206,14 @@ module RailsMind
|
|
|
147
206
|
RailsMind.client.emit(:request, "#{payload[:controller]}##{payload[:action]}", duration_ms: event.duration,
|
|
148
207
|
status: status.to_s, properties: properties)
|
|
149
208
|
end
|
|
209
|
+
subscribe("start_processing.action_controller") do |event|
|
|
210
|
+
Visitor.seed_daily(event.payload[:request])
|
|
211
|
+
end
|
|
212
|
+
subscribe("process_action.action_controller") do |event|
|
|
213
|
+
client = RailsMind.client
|
|
214
|
+
config = client.config
|
|
215
|
+
PageView.record(event.payload, client) if client.collector && config.analytics && config.page_views
|
|
216
|
+
end
|
|
150
217
|
subscribe("sql.active_record") do |event|
|
|
151
218
|
queries = Thread.current[:rails_mind_queries]
|
|
152
219
|
next unless queries && !event.payload[:cached] && !%w[SCHEMA TRANSACTION].include?(event.payload[:name])
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
require_relative "visitor"
|
|
3
|
+
|
|
4
|
+
module RailsMind
|
|
5
|
+
# One HTML page a controller rendered for a person: a successful GET, not a
|
|
6
|
+
# Turbo Frame, a prefetch, an XHR, a health check or a known bot. Built from
|
|
7
|
+
# the process_action notification, so it shares the request event's trace
|
|
8
|
+
# and the identity the app set during the action.
|
|
9
|
+
module PageView
|
|
10
|
+
# The wire contract with the hosted app. Every key survives both redactors.
|
|
11
|
+
PROPERTIES = %w[route path host referrer utm_source utm_medium utm_campaign utm_term utm_content ref
|
|
12
|
+
user_agent country visitor_id_kind].freeze
|
|
13
|
+
TAGS = %w[utm_source utm_medium utm_campaign utm_term utm_content ref].freeze
|
|
14
|
+
ROUTE_LIMIT = 500
|
|
15
|
+
PATH_LIMIT = 1_000
|
|
16
|
+
HOST_LIMIT = 255
|
|
17
|
+
REFERRER_LIMIT = 1_000
|
|
18
|
+
TAG_LIMIT = 200
|
|
19
|
+
USER_AGENT_LIMIT = 500
|
|
20
|
+
COUNTRY_HEADERS = %w[HTTP_CF_IPCOUNTRY HTTP_CLOUDFRONT_VIEWER_COUNTRY].freeze
|
|
21
|
+
# Cloudflare's unknown country and Tor.
|
|
22
|
+
NO_COUNTRY = %w[XX T1].freeze
|
|
23
|
+
PREFETCH_HEADERS = %w[HTTP_SEC_PURPOSE HTTP_X_SEC_PURPOSE].freeze
|
|
24
|
+
NO_CONTENT = [ 204, 205 ].freeze
|
|
25
|
+
FRAMEWORK_CONTROLLER = /\A(?:Rails|ActiveStorage)::/
|
|
26
|
+
# Crawlers, link previews, LLM fetchers, monitors and bare HTTP clients. A
|
|
27
|
+
# blank user agent is not counted either. "cloudfront" is what CloudFront
|
|
28
|
+
# sends when a distribution does not forward the caller's user agent.
|
|
29
|
+
BOT = Regexp.new([
|
|
30
|
+
"bot", "crawl", "spider", "slurp", "scrap", "fetch", "monitor", "preview",
|
|
31
|
+
"headless", "phantom", "selenium", "puppeteer", "playwright",
|
|
32
|
+
"curl", "wget", "python-requests", "httpclient", "okhttp", "go-http-client", "java/",
|
|
33
|
+
"facebookexternalhit", "twitterbot", "slackbot", "discordbot", "linkedinbot", "telegrambot", "whatsapp",
|
|
34
|
+
"googlebot", "bingbot", "duckduckbot", "yandex", "baidu", "sogou", "seznam", "petalbot", "applebot",
|
|
35
|
+
"ahrefs", "semrush", "mj12", "dotbot", "sistrix", "barkrowler", "serpstat", "dataforseo", "blexbot", "exabot",
|
|
36
|
+
"gptbot", "chatgpt-user", "oai-searchbot", "claudebot", "claude-web", "anthropic", "cohere-ai",
|
|
37
|
+
"perplexitybot", "amazonbot", "bytespider", "google-extended", "meta-externalagent",
|
|
38
|
+
"cloudfront"
|
|
39
|
+
].map { |word| Regexp.escape(word) }.join("|"), Regexp::IGNORECASE)
|
|
40
|
+
|
|
41
|
+
class << self
|
|
42
|
+
def record(payload, client = RailsMind.client)
|
|
43
|
+
request, response = payload[:request], payload[:response]
|
|
44
|
+
return false unless request && response
|
|
45
|
+
config = client.config
|
|
46
|
+
return false unless counted?(payload, request, response, config)
|
|
47
|
+
properties = properties(request)
|
|
48
|
+
attributes = {}
|
|
49
|
+
settable = settable?(response)
|
|
50
|
+
if cookie_visitor?(request, settable, config)
|
|
51
|
+
properties["visitor_id_kind"] = "cookie"
|
|
52
|
+
request.set_header(Visitor::WRITE, { secure: request.ssl? }) if settable
|
|
53
|
+
else
|
|
54
|
+
properties["visitor_id_kind"] = "daily_hash"
|
|
55
|
+
attributes[:visitor_id] = Visitor.daily_id(request)
|
|
56
|
+
end
|
|
57
|
+
attributes.merge!(user_id: nil, account_id: nil) unless config.link_page_views_to_users
|
|
58
|
+
client.emit(:pageview, "#{payload[:controller]}##{payload[:action]}", properties: properties, **attributes)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def counted?(payload, request, response, config)
|
|
64
|
+
return false unless (payload[:method] || request.request_method) == "GET"
|
|
65
|
+
return false unless (200..299).cover?(payload[:status].to_i)
|
|
66
|
+
return false unless html?(payload, response)
|
|
67
|
+
return false unless request.get_header("HTTP_TURBO_FRAME").to_s.empty?
|
|
68
|
+
return false if PREFETCH_HEADERS.any? { |header| request.get_header(header).to_s.downcase.include?("prefetch") }
|
|
69
|
+
return false if request.xhr?
|
|
70
|
+
return false if payload[:controller].to_s.match?(FRAMEWORK_CONTROLLER)
|
|
71
|
+
user_agent = request.user_agent.to_s
|
|
72
|
+
return false if user_agent.strip.empty? || user_agent.match?(BOT)
|
|
73
|
+
!ignored?(config, request)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Rails types a streamed template only when the response commits, as
|
|
77
|
+
# text/html like any response still untyped. An HTML request answered
|
|
78
|
+
# with content and no type yet is such a page.
|
|
79
|
+
def html?(payload, response)
|
|
80
|
+
type = response.media_type
|
|
81
|
+
return type == "text/html" if type
|
|
82
|
+
payload[:format] == :html && !NO_CONTENT.include?(payload[:status].to_i)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# The cookie ID the request boundary put in context, unless the response
|
|
86
|
+
# cannot take the cookie and the visitor brought none: a fresh ID there
|
|
87
|
+
# could never be set, so that page view counts by the daily hash.
|
|
88
|
+
def cookie_visitor?(request, settable, config)
|
|
89
|
+
return false unless config.visitor_cookie
|
|
90
|
+
return false if !settable && !request.get_header(Visitor::ENV_COOKIE)
|
|
91
|
+
!Context.current[:visitor_id].to_s.empty?
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# A hook that raises has not said the page view counts.
|
|
95
|
+
def ignored?(config, request)
|
|
96
|
+
hook = config.ignore_page_view
|
|
97
|
+
hook ? hook.call(request) : false
|
|
98
|
+
rescue StandardError
|
|
99
|
+
true
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Not a response a shared cache may keep, nor one already on its way:
|
|
103
|
+
# ActionController::Live sends the headers at the first write or render,
|
|
104
|
+
# before the action ends.
|
|
105
|
+
def settable?(response)
|
|
106
|
+
return false if response.respond_to?(:committed?) && response.committed?
|
|
107
|
+
!public?(response)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def public?(response)
|
|
111
|
+
control = response.cache_control if response.respond_to?(:cache_control)
|
|
112
|
+
return true if control.is_a?(Hash) && control[:public]
|
|
113
|
+
response.get_header("Cache-Control").to_s.match?(Visitor::PUBLIC)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def properties(request)
|
|
117
|
+
values = {}
|
|
118
|
+
route = route(request)
|
|
119
|
+
values["route"] = route if route
|
|
120
|
+
values["path"] = request.filtered_path.to_s[0, PATH_LIMIT]
|
|
121
|
+
values["host"] = request.host.to_s.downcase[0, HOST_LIMIT]
|
|
122
|
+
referrer = referrer(request.referer)
|
|
123
|
+
values["referrer"] = referrer if referrer
|
|
124
|
+
values.merge!(tags(request))
|
|
125
|
+
values["user_agent"] = request.user_agent.to_s[0, USER_AGENT_LIMIT]
|
|
126
|
+
country = country(request)
|
|
127
|
+
values["country"] = country if country
|
|
128
|
+
values
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Inside a mounted engine Rails gives the engine's own pattern, so the
|
|
132
|
+
# mount path (the request's script name) goes in front, as in the path.
|
|
133
|
+
def route(request)
|
|
134
|
+
return unless request.respond_to?(:route_uri_pattern)
|
|
135
|
+
pattern = request.route_uri_pattern.to_s.delete_suffix("(.:format)")
|
|
136
|
+
return if pattern.empty?
|
|
137
|
+
pattern = "#{request.script_name}#{pattern}".chomp("/")
|
|
138
|
+
(pattern.empty? ? "/" : pattern)[0, ROUTE_LIMIT]
|
|
139
|
+
rescue StandardError
|
|
140
|
+
nil
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Scheme, host and path only: a referring URL's query and fragment stay home.
|
|
144
|
+
def referrer(value)
|
|
145
|
+
return unless value.is_a?(String) && !value.strip.empty?
|
|
146
|
+
uri = URI.parse(value.strip)
|
|
147
|
+
return unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
|
|
148
|
+
"#{uri.scheme.downcase}://#{uri.host.downcase}#{uri.path}"[0, REFERRER_LIMIT]
|
|
149
|
+
rescue URI::Error
|
|
150
|
+
nil
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# A tag the app's filter_parameters would mask in the path is not sent.
|
|
154
|
+
def tags(request)
|
|
155
|
+
query = request.GET
|
|
156
|
+
raw = TAGS.each_with_object({}) { |key, values| values[key] = query[key] if query[key].is_a?(String) }
|
|
157
|
+
filtered = request.parameter_filter.filter(raw)
|
|
158
|
+
raw.each_with_object({}) do |(key, value), tags|
|
|
159
|
+
next unless filtered[key] == value
|
|
160
|
+
value = value.strip
|
|
161
|
+
tags[key] = value[0, TAG_LIMIT] unless value.empty?
|
|
162
|
+
end
|
|
163
|
+
rescue StandardError
|
|
164
|
+
{}
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def country(request)
|
|
168
|
+
COUNTRY_HEADERS.each do |header|
|
|
169
|
+
code = request.get_header(header).to_s.strip.upcase
|
|
170
|
+
return code if code.match?(/\A[A-Z]{2}\z/) && !NO_COUNTRY.include?(code)
|
|
171
|
+
end
|
|
172
|
+
nil
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
data/lib/rails_mind/redactor.rb
CHANGED
|
@@ -2,13 +2,22 @@ module RailsMind
|
|
|
2
2
|
class Redactor
|
|
3
3
|
FILTER = /password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization|cookie|credential|session|email|phone|address|credit|card|body|params|arguments|bind|sql|db\.statement|query\.text/i
|
|
4
4
|
MAX_STRING = 1024
|
|
5
|
+
# Scrubbing reads at most this much of a string; what it sends is still at
|
|
6
|
+
# most MAX_STRING. Every pass below is linear, and the cap bounds the rest.
|
|
7
|
+
INPUT_LIMIT = 8192
|
|
8
|
+
# Identity, correlation and release values are not free text: a numeric
|
|
9
|
+
# account ID that passes the Luhn check is not a card number.
|
|
10
|
+
PLAIN_FIELDS = %w[id kind occurred_at status release trace_id span_id parent_span_id request_id job_id visitor_id user_id account_id].freeze
|
|
5
11
|
|
|
6
12
|
def call(value, depth = 0)
|
|
7
13
|
return "[TRUNCATED]" if depth > 5
|
|
8
14
|
case value
|
|
9
15
|
when Hash
|
|
10
16
|
value.first(50).to_h.transform_keys { |key| key.to_s[0, 100] }.each_with_object({}) do |(key, item), clean|
|
|
11
|
-
clean[key] = key.match?(FILTER)
|
|
17
|
+
clean[key] = if key.match?(FILTER) then "[FILTERED]"
|
|
18
|
+
elsif depth.zero? && PLAIN_FIELDS.include?(key) && (item.is_a?(String) || item.is_a?(Symbol)) then scrub(item.to_s, free_text: false)
|
|
19
|
+
else call(item, depth + 1)
|
|
20
|
+
end
|
|
12
21
|
end
|
|
13
22
|
when Array then value.first(50).map { |item| call(item, depth + 1) }
|
|
14
23
|
when String then scrub(value)
|
|
@@ -19,30 +28,56 @@ module RailsMind
|
|
|
19
28
|
end
|
|
20
29
|
end
|
|
21
30
|
|
|
31
|
+
# An authorization scheme names the credential that follows it, so the
|
|
32
|
+
# scheme and the credential go together. Digest and AWS signature headers
|
|
33
|
+
# carry several credential parts, so their whole value goes.
|
|
34
|
+
SCHEME = "(?:bearer|basic|token|negotiate|ntlm|bot|apikey)\\s+".freeze
|
|
35
|
+
WHOLE_VALUE_SCHEME = "(?:digest|aws4-hmac-sha256)\\s[^\\n]*".freeze
|
|
36
|
+
|
|
22
37
|
# Exception messages carry what generic scrubbing cannot recognize: database
|
|
23
38
|
# errors quote the row, key or input that failed (and the SQL with its
|
|
24
|
-
# literals), and Ruby prints hashes and keyword arguments with their values
|
|
39
|
+
# literals), and Ruby prints hashes and keyword arguments with their values,
|
|
40
|
+
# header values wrapped in arrays (Net::HTTP#to_hash) included.
|
|
25
41
|
DATABASE_ERROR = /\A(?:PG|Mysql2|Trilogy|SQLite3)::|StatementInvalid|RecordNotUnique|NotNullViolation|ForeignKeyViolation|ValueTooLong|RangeError\z/
|
|
26
|
-
SENSITIVE_VALUE = /((?:"|:)?(?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization|credential|cookie|session)\w
|
|
42
|
+
SENSITIVE_VALUE = /((?:"|:)?(?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization|credential|cookie|session|phone|ssn)\w{0,64}"?\s*(?:=>|:)\s*(?:\[(?!FILTERED\]))?)(?:\[FILTERED\]|"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|#{WHOLE_VALUE_SCHEME}|(?:#{SCHEME})?[^\s,}\])]+)/i
|
|
43
|
+
|
|
44
|
+
# PostgreSQL connections use standard_conforming_strings, where a backslash
|
|
45
|
+
# does not escape the quote of a '...' literal (it does inside E'...');
|
|
46
|
+
# MySQL lets it escape the quote. Text with a backslash is read both ways,
|
|
47
|
+
# and anything either reading takes for a literal goes.
|
|
48
|
+
STANDARD_LITERAL = /(?<!\w)[Ee]'(?:[^'\\]|\\.|'')*'?|'(?:[^']|'')*'?/
|
|
49
|
+
ESCAPED_LITERAL = /'(?:[^'\\]|\\.|'')*'?/
|
|
27
50
|
|
|
28
51
|
def scrub_message(error)
|
|
29
|
-
text = error.message.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�")
|
|
52
|
+
text = bounded(error.message.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�"))
|
|
30
53
|
text = text.gsub(SENSITIVE_VALUE, '\\1"[FILTERED]"')
|
|
31
54
|
.gsub(/(Failing row contains )\([^\n]*\)/, "\\1(…)")
|
|
32
55
|
.gsub(/(Key \([^)\n]*\)=)\([^\n]*\)/, "\\1(…)")
|
|
33
56
|
.gsub(/(invalid input (?:syntax|value) for [^:\n]*: )"[^"\n]*"/, '\\1"…"')
|
|
34
|
-
text = text
|
|
57
|
+
text = mask_literals(text) if database_error?(error)
|
|
35
58
|
scrub(text)
|
|
36
59
|
end
|
|
37
60
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
61
|
+
# Userinfo runs to the last @ before the path, so an unencoded @ in a
|
|
62
|
+
# password cannot leave its tail behind.
|
|
63
|
+
USERINFO = %r{\b([a-z][a-z0-9+.\-]{0,31}://)[^\s/]*@}i
|
|
64
|
+
# Plain or percent-encoded, as a GET form puts an address in a query string.
|
|
65
|
+
EMAIL = /\b[A-Z0-9._%+\-]{1,128}(?:@|%40)[A-Z0-9.\-]{1,253}\.[A-Z]{2,}\b/i
|
|
66
|
+
AUTHORIZATION = /((?:proxy-)?authorization["']?\s*(?:=>|[=:])\s*(?:\[(?!FILTERED\]))?\s*["']?)(?:\[FILTERED\]|#{WHOLE_VALUE_SCHEME}|(?:#{SCHEME})?[^\s,;&"']+)/i
|
|
67
|
+
KEY_VALUE = /((?:password|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization)\s*(?:=>|[=:])\s*)(?:#{SCHEME})?[^\s,;&]+/i
|
|
68
|
+
PROVIDER_TOKEN = /\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_\w{22,}|(?:AKIA|ASIA)[A-Z0-9]{16}\b|[sr]k_(?:live|test)_[A-Za-z0-9]{10,}|xox[abposr]-[A-Za-z0-9-]{10,}|sk-(?:proj-|ant-)?[A-Za-z0-9_-]{20,}|eyJ[\w-]{5,512}\.[\w-]{5,}\.[\w-]*)/
|
|
69
|
+
SSN = /\b\d{3}([- ])\d{2}\1\d{4}\b/
|
|
70
|
+
DIGIT_RUN = /(?<!\w)\d(?:[ -]?\d)*(?!\w)/
|
|
71
|
+
|
|
72
|
+
def scrub(value, free_text: true)
|
|
73
|
+
text = bounded(value.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�"))
|
|
74
|
+
.gsub(USERINFO, '\\1[FILTERED]@')
|
|
75
|
+
.gsub(EMAIL, "[EMAIL]")
|
|
76
|
+
.gsub(AUTHORIZATION, '\\1[FILTERED]')
|
|
41
77
|
.gsub(/\bBearer\s+\S+/i, "Bearer [FILTERED]")
|
|
42
|
-
.gsub(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
.slice(0, MAX_STRING)
|
|
78
|
+
.gsub(KEY_VALUE, '\\1[FILTERED]')
|
|
79
|
+
text = text.gsub(PROVIDER_TOKEN, "[FILTERED]").gsub(SSN, "[FILTERED]").gsub(DIGIT_RUN) { |run| mask_cards(run) } if free_text
|
|
80
|
+
text.gsub(%r{https?://\S+}) { |url| url[/\A[^?#]*/] }.slice(0, MAX_STRING)
|
|
46
81
|
end
|
|
47
82
|
|
|
48
83
|
private
|
|
@@ -50,5 +85,85 @@ module RailsMind
|
|
|
50
85
|
def database_error?(error)
|
|
51
86
|
error.class.ancestors.any? { |ancestor| ancestor.name.to_s.match?(DATABASE_ERROR) }
|
|
52
87
|
end
|
|
88
|
+
|
|
89
|
+
# A cut string ends at its last whitespace, so no half token is left that a
|
|
90
|
+
# pass could no longer recognize.
|
|
91
|
+
def bounded(text)
|
|
92
|
+
return text if text.length <= INPUT_LIMIT
|
|
93
|
+
text = text[0, INPUT_LIMIT]
|
|
94
|
+
cut = text.rindex(/\s/)
|
|
95
|
+
cut ? text[0, cut] : text
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def mask_literals(text)
|
|
99
|
+
return text.gsub(ESCAPED_LITERAL, "?") unless text.include?("\\")
|
|
100
|
+
spans = [ STANDARD_LITERAL, ESCAPED_LITERAL ].flat_map { |pattern| text.to_enum(:scan, pattern).map { Regexp.last_match.offset(0) } }
|
|
101
|
+
output, position = +"", 0
|
|
102
|
+
merge(spans).each do |start, finish|
|
|
103
|
+
output << text[position...start] << "?"
|
|
104
|
+
position = finish
|
|
105
|
+
end
|
|
106
|
+
output << text[position..]
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def merge(spans)
|
|
110
|
+
spans.sort.each_with_object([]) do |(start, finish), merged|
|
|
111
|
+
if merged.any? && start < merged.last[1]
|
|
112
|
+
merged.last[1] = [ merged.last[1], finish ].max
|
|
113
|
+
else
|
|
114
|
+
merged << [ start, finish ]
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# A card number is 13 to 19 digits, maybe in groups split by spaces or
|
|
120
|
+
# dashes, that pass the Luhn check. A run of digit groups can hold one next
|
|
121
|
+
# to other numbers (an expiry, a quantity), so every span of whole groups
|
|
122
|
+
# with at most 19 digits is tried, and the longest that passes goes.
|
|
123
|
+
def mask_cards(run)
|
|
124
|
+
groups = run.to_enum(:scan, /\d+/).map { Regexp.last_match.offset(0) }
|
|
125
|
+
digits = run.delete(" -")
|
|
126
|
+
return run if digits.length < 13
|
|
127
|
+
bounds, count = [], 0
|
|
128
|
+
groups.each { |start, finish| bounds << [ count, count += finish - start ] }
|
|
129
|
+
plain, doubled = luhn_sums(digits)
|
|
130
|
+
output, position, first = +"", 0, 0
|
|
131
|
+
while first < groups.size
|
|
132
|
+
longest, from = nil, bounds[first][0]
|
|
133
|
+
(first...groups.size).each do |last|
|
|
134
|
+
to = bounds[last][1]
|
|
135
|
+
break if to - from > 19
|
|
136
|
+
next if to - from < 13
|
|
137
|
+
parity = to % 2
|
|
138
|
+
sum = doubled[parity][to] - doubled[parity][from] + plain[1 - parity][to] - plain[1 - parity][from]
|
|
139
|
+
longest = last if (sum % 10).zero?
|
|
140
|
+
end
|
|
141
|
+
if longest
|
|
142
|
+
output << run[position...groups[first][0]] << "[FILTERED]"
|
|
143
|
+
position, first = groups[longest][1], longest + 1
|
|
144
|
+
else
|
|
145
|
+
first += 1
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
output << run[position..]
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Prefix sums of each digit, plain and Luhn-doubled, split by position
|
|
152
|
+
# parity. In a span ending at index `to`, Luhn doubles the digits whose
|
|
153
|
+
# position has the parity of `to`, so any span's checksum is four lookups.
|
|
154
|
+
def luhn_sums(digits)
|
|
155
|
+
plain, doubled = Array.new(2) { [ 0 ] }, Array.new(2) { [ 0 ] }
|
|
156
|
+
digits.each_char.with_index do |char, index|
|
|
157
|
+
value = char.ord - 48
|
|
158
|
+
twice = value * 2
|
|
159
|
+
twice -= 9 if twice > 9
|
|
160
|
+
[ 0, 1 ].each do |parity|
|
|
161
|
+
own = index % 2 == parity
|
|
162
|
+
plain[parity] << plain[parity].last + (own ? value : 0)
|
|
163
|
+
doubled[parity] << doubled[parity].last + (own ? twice : 0)
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
[ plain, doubled ]
|
|
167
|
+
end
|
|
53
168
|
end
|
|
54
169
|
end
|
data/lib/rails_mind/transport.rb
CHANGED
|
@@ -18,6 +18,8 @@ module RailsMind
|
|
|
18
18
|
def send_batch(json)
|
|
19
19
|
request = Net::HTTP::Post.new(@uri)
|
|
20
20
|
request.body = json
|
|
21
|
+
zone = time_zone
|
|
22
|
+
request["X-RailsMind-Time-Zone"] = zone if zone
|
|
21
23
|
perform(request)
|
|
22
24
|
end
|
|
23
25
|
|
|
@@ -29,6 +31,18 @@ module RailsMind
|
|
|
29
31
|
|
|
30
32
|
private
|
|
31
33
|
|
|
34
|
+
# The app's IANA zone (Rails' default "UTC" is Etc/UTC), so RailsMind counts
|
|
35
|
+
# days where the app's people live. Unknown until Rails has set it.
|
|
36
|
+
def time_zone
|
|
37
|
+
@time_zone ||= begin
|
|
38
|
+
zone = Time.zone_default if Time.respond_to?(:zone_default)
|
|
39
|
+
name = zone&.tzinfo&.name
|
|
40
|
+
name if name.is_a?(String) && !name.empty?
|
|
41
|
+
rescue StandardError
|
|
42
|
+
nil
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
32
46
|
def perform(request)
|
|
33
47
|
Context.suppress do
|
|
34
48
|
request["Authorization"] = "Bearer #{@config.token}"
|
data/lib/rails_mind/version.rb
CHANGED