rails_mind 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,8 +39,14 @@ module RailsMind
39
39
  Thread.current[:rails_mind_context] = previous
40
40
  end
41
41
 
42
- def identify(user_id: nil, account_id: nil, visitor_id: nil)
43
- current.merge!(user_id: user_id&.to_s, account_id: account_id&.to_s, visitor_id: visitor_id&.to_s)
42
+ UNSET = Object.new.freeze
43
+
44
+ # Signing in names the user and account; the visitor stays the one this
45
+ # request already carries unless visitor_id is passed (nil clears it).
46
+ def identify(user_id: nil, account_id: nil, visitor_id: UNSET)
47
+ values = { user_id: user_id&.to_s, account_id: account_id&.to_s }
48
+ values[:visitor_id] = visitor_id&.to_s unless UNSET.equal?(visitor_id)
49
+ current.merge!(values)
44
50
  end
45
51
 
46
52
  def suppressed? = Thread.current[:rails_mind_suppressed] == true
@@ -22,19 +22,34 @@ module RailsMind
22
22
  release_present: !@config.release.to_s.strip.empty?,
23
23
  enabled: @config.enabled?,
24
24
  automatic_sampling: @config.automatic_sampling,
25
+ page_views: @config.page_views,
26
+ visitor_cookie: @config.visitor_cookie,
27
+ link_page_views_to_users: @config.link_page_views_to_users,
25
28
  plan: [
26
29
  "Add one initializer. No database migrations or Gemfile changes.",
27
30
  "Requests: automatic sampling of routine successes; jobs via Rails notifications and errors via Rails.error stay unsampled.",
28
- detected["ahoy_matey"] ? "Ahoy detected: keep current store; explicitly opt into mirror mode after review." : "Analytics: optionally add ahoy_matey; choose hosted-only store or existing database store.",
31
+ "Page views: HTML pages your controllers render are counted with a first-party visitor cookie; config.page_views, config.visitor_cookie and config.link_page_views_to_users turn parts off.",
32
+ ("Ahoy detected: RailsMind no longer reads Ahoy and counts page views itself. Remove any require \"rails_mind/integrations/ahoy\" and RailsMind::Integrations.mirror_ahoy! call, or the app fails at boot; Ahoy keeps its own tables." if detected["ahoy_matey"]),
29
33
  detected["opentelemetry-sdk"] ? "OpenTelemetry detected: attach RailsMind exporter after existing SDK setup; preserve sampler and exporters." : "APM traces: optionally add selected OpenTelemetry libraries and explicitly configure one provider.",
30
34
  detected["flipper"] ? "Flipper detected: observe notifications only; preserve adapter, groups, targeting and cache." : "Flags: Flipper is optional. No flag configuration is created.",
31
35
  "Existing error reporters stay installed. RailsMind deduplicates its own collector only.",
32
- "Browser: optional Ahoy adapter; enable only after checking existing pageview handlers.",
33
36
  "Business/error signals are not sampled; routine requests follow the hosted policy. Local opt-out does not disable server sampling or quotas. Inspect RailsMind.stats for queue losses and unconfirmed delivery."
34
- ],
37
+ ].compact,
35
38
  limitations: [ "Compatibility applies to tested versions only; this is not a historical import.",
36
39
  "No network check is performed and credential values are never printed." ]
37
40
  }
38
41
  end
42
+
43
+ # What keeps collection off, one sentence each, naming the setting to change.
44
+ def problems
45
+ [].tap do |found|
46
+ if @config.token.to_s.strip.empty?
47
+ found << "RAILS_MIND_KEY is not set, so collection is off. Set it in this app's environment to the ingest key from RailsMind Setup, restart, then run bin/rails rails_mind:verify."
48
+ end
49
+ if @config.endpoint.to_s.strip.empty?
50
+ found << "The RailsMind endpoint is blank, so collection is off. Leave config.endpoint and RAILS_MIND_ENDPOINT unset to use #{Configuration::DEFAULT_ENDPOINT}, or set your RailsMind instance's URL."
51
+ end
52
+ end
53
+ end
39
54
  end
40
55
  end
@@ -1,6 +1,8 @@
1
1
  require "active_support/notifications"
2
2
  require "active_support/inflector"
3
3
  require "digest"
4
+ require_relative "../visitor"
5
+ require_relative "../page_view"
4
6
 
5
7
  module RailsMind
6
8
  module Integrations
@@ -44,7 +46,10 @@ module RailsMind
44
46
  Context.with_trace do
45
47
  previous = Thread.current[:rails_mind_queries]
46
48
  Thread.current[:rails_mind_queries] = { count: 0, fingerprints: Hash.new(0), shapes: {} }
47
- @app.call(env)
49
+ Visitor.start(env)
50
+ response = @app.call(env)
51
+ Visitor.finish(env, response)
52
+ response
48
53
  rescue StandardError => error
49
54
  # Capture identity before this request boundary unwinds. Object
50
55
  # dedup still leaves other error reporters unaffected.
@@ -113,18 +118,72 @@ module RailsMind
113
118
  # was cut), numbers and hex literals become ?, and comments go. Double
114
119
  # quotes are identifiers, except on MySQL, where they can be strings.
115
120
  LEXEME = /(?<!\w)[Ee]'(?:[^'\\]|\\.|'')*'?|'(?:[^'\\]|\\.|'')*'?|\$(\w*)\$.*?(?:\$\1\$|\z)|"(?:[^"\\]|\\.|"")*"?|`[^`]*`?|--[^\n]*|\/\*.*?(?:\*\/|\z)|\b0[xX]\h+\b|\b\d+(?:\.\d+)?(?:[eE][-+]?\d+)?\b/m
121
+ # The same lexemes where a backslash escapes nothing outside E'...', as on
122
+ # PostgreSQL (Rails turns standard_conforming_strings on) and SQLite. Which
123
+ # rule applies is not always known (MySQL can turn backslash escapes off),
124
+ # so a statement with a backslash is read both ways and anything either
125
+ # reading takes for a literal goes. Without a backslash both agree.
126
+ STANDARD_LEXEME = /(?<!\w)[Ee]'(?:[^'\\]|\\.|'')*'?|'(?:[^']|'')*'?|\$(\w*)\$.*?(?:\$\1\$|\z)|"(?:[^"]|"")*"?|`[^`]*`?|--[^\n]*|\/\*.*?(?:\*\/|\z)|\b0[xX]\h+\b|\b\d+(?:\.\d+)?(?:[eE][-+]?\d+)?\b/m
116
127
 
117
128
  def self.normalize(sql, mysql: false)
118
- sql.scrub("?").gsub(LEXEME) do |token|
119
- case token[0]
120
- when "-", "/" then " "
121
- when "`" then token
122
- when '"' then mysql ? "?" : token
123
- else "?"
124
- end
125
- end.gsub(/\s+/, " ").strip
129
+ sql = sql.scrub("?")
130
+ text = sql.include?("\\") ? read_both_ways(sql, mysql) : sql.gsub(LEXEME) { |token| replacement(token, mysql) }
131
+ text.gsub(/\s+/, " ").strip
126
132
  end
127
133
 
134
+ def self.replacement(token, mysql)
135
+ case token[0]
136
+ when "-", "/" then " "
137
+ when "`" then token
138
+ when '"' then mysql ? "?" : token
139
+ else "?"
140
+ end
141
+ end
142
+
143
+ # Literal spans from either reading merge into one ?; a comment that
144
+ # overlaps one joins it. Kept identifiers stay only where no reading saw a
145
+ # literal.
146
+ def self.read_both_ways(sql, mysql)
147
+ literals, comments = [], []
148
+ [ STANDARD_LEXEME, LEXEME ].each do |pattern|
149
+ sql.scan(pattern) do
150
+ token = Regexp.last_match
151
+ case replacement(token[0], mysql)
152
+ when "?" then literals << token.offset(0)
153
+ when " " then comments << token.offset(0)
154
+ end
155
+ end
156
+ end
157
+ literals = merge(literals)
158
+ loop do
159
+ # Merged literal spans are sorted and disjoint, so their ends are too.
160
+ joining, comments = comments.partition do |start, finish|
161
+ overlap = literals.bsearch { |_, last| last > start }
162
+ overlap && overlap[0] < finish
163
+ end
164
+ break if joining.empty?
165
+ literals = merge(literals + joining)
166
+ end
167
+ spans = (literals.map { |span| span + [ "?" ] } + merge(comments).map { |span| span + [ " " ] }).sort
168
+ output, position = +"", 0
169
+ spans.each do |start, finish, mark|
170
+ output << sql[position...start] << mark
171
+ position = finish
172
+ end
173
+ output << sql[position..]
174
+ end
175
+
176
+ def self.merge(spans)
177
+ spans.sort.each_with_object([]) do |(start, finish), merged|
178
+ if merged.any? && start < merged.last[1]
179
+ merged.last[1] = [ merged.last[1], finish ].max
180
+ else
181
+ merged << [ start, finish ]
182
+ end
183
+ end
184
+ end
185
+ private_class_method :replacement, :read_both_ways, :merge
186
+
128
187
  class << self
129
188
  def install!
130
189
  return if @subscriptions
@@ -147,6 +206,14 @@ module RailsMind
147
206
  RailsMind.client.emit(:request, "#{payload[:controller]}##{payload[:action]}", duration_ms: event.duration,
148
207
  status: status.to_s, properties: properties)
149
208
  end
209
+ subscribe("start_processing.action_controller") do |event|
210
+ Visitor.seed_daily(event.payload[:request])
211
+ end
212
+ subscribe("process_action.action_controller") do |event|
213
+ client = RailsMind.client
214
+ config = client.config
215
+ PageView.record(event.payload, client) if client.collector && config.analytics && config.page_views
216
+ end
150
217
  subscribe("sql.active_record") do |event|
151
218
  queries = Thread.current[:rails_mind_queries]
152
219
  next unless queries && !event.payload[:cached] && !%w[SCHEMA TRANSACTION].include?(event.payload[:name])
@@ -0,0 +1,176 @@
1
+ require "uri"
2
+ require_relative "visitor"
3
+
4
+ module RailsMind
5
+ # One HTML page a controller rendered for a person: a successful GET, not a
6
+ # Turbo Frame, a prefetch, an XHR, a health check or a known bot. Built from
7
+ # the process_action notification, so it shares the request event's trace
8
+ # and the identity the app set during the action.
9
+ module PageView
10
+ # The wire contract with the hosted app. Every key survives both redactors.
11
+ PROPERTIES = %w[route path host referrer utm_source utm_medium utm_campaign utm_term utm_content ref
12
+ user_agent country visitor_id_kind].freeze
13
+ TAGS = %w[utm_source utm_medium utm_campaign utm_term utm_content ref].freeze
14
+ ROUTE_LIMIT = 500
15
+ PATH_LIMIT = 1_000
16
+ HOST_LIMIT = 255
17
+ REFERRER_LIMIT = 1_000
18
+ TAG_LIMIT = 200
19
+ USER_AGENT_LIMIT = 500
20
+ COUNTRY_HEADERS = %w[HTTP_CF_IPCOUNTRY HTTP_CLOUDFRONT_VIEWER_COUNTRY].freeze
21
+ # Cloudflare's unknown country and Tor.
22
+ NO_COUNTRY = %w[XX T1].freeze
23
+ PREFETCH_HEADERS = %w[HTTP_SEC_PURPOSE HTTP_X_SEC_PURPOSE].freeze
24
+ NO_CONTENT = [ 204, 205 ].freeze
25
+ FRAMEWORK_CONTROLLER = /\A(?:Rails|ActiveStorage)::/
26
+ # Crawlers, link previews, LLM fetchers, monitors and bare HTTP clients. A
27
+ # blank user agent is not counted either. "cloudfront" is what CloudFront
28
+ # sends when a distribution does not forward the caller's user agent.
29
+ BOT = Regexp.new([
30
+ "bot", "crawl", "spider", "slurp", "scrap", "fetch", "monitor", "preview",
31
+ "headless", "phantom", "selenium", "puppeteer", "playwright",
32
+ "curl", "wget", "python-requests", "httpclient", "okhttp", "go-http-client", "java/",
33
+ "facebookexternalhit", "twitterbot", "slackbot", "discordbot", "linkedinbot", "telegrambot", "whatsapp",
34
+ "googlebot", "bingbot", "duckduckbot", "yandex", "baidu", "sogou", "seznam", "petalbot", "applebot",
35
+ "ahrefs", "semrush", "mj12", "dotbot", "sistrix", "barkrowler", "serpstat", "dataforseo", "blexbot", "exabot",
36
+ "gptbot", "chatgpt-user", "oai-searchbot", "claudebot", "claude-web", "anthropic", "cohere-ai",
37
+ "perplexitybot", "amazonbot", "bytespider", "google-extended", "meta-externalagent",
38
+ "cloudfront"
39
+ ].map { |word| Regexp.escape(word) }.join("|"), Regexp::IGNORECASE)
40
+
41
+ class << self
42
+ def record(payload, client = RailsMind.client)
43
+ request, response = payload[:request], payload[:response]
44
+ return false unless request && response
45
+ config = client.config
46
+ return false unless counted?(payload, request, response, config)
47
+ properties = properties(request)
48
+ attributes = {}
49
+ settable = settable?(response)
50
+ if cookie_visitor?(request, settable, config)
51
+ properties["visitor_id_kind"] = "cookie"
52
+ request.set_header(Visitor::WRITE, { secure: request.ssl? }) if settable
53
+ else
54
+ properties["visitor_id_kind"] = "daily_hash"
55
+ attributes[:visitor_id] = Visitor.daily_id(request)
56
+ end
57
+ attributes.merge!(user_id: nil, account_id: nil) unless config.link_page_views_to_users
58
+ client.emit(:pageview, "#{payload[:controller]}##{payload[:action]}", properties: properties, **attributes)
59
+ end
60
+
61
+ private
62
+
63
+ def counted?(payload, request, response, config)
64
+ return false unless (payload[:method] || request.request_method) == "GET"
65
+ return false unless (200..299).cover?(payload[:status].to_i)
66
+ return false unless html?(payload, response)
67
+ return false unless request.get_header("HTTP_TURBO_FRAME").to_s.empty?
68
+ return false if PREFETCH_HEADERS.any? { |header| request.get_header(header).to_s.downcase.include?("prefetch") }
69
+ return false if request.xhr?
70
+ return false if payload[:controller].to_s.match?(FRAMEWORK_CONTROLLER)
71
+ user_agent = request.user_agent.to_s
72
+ return false if user_agent.strip.empty? || user_agent.match?(BOT)
73
+ !ignored?(config, request)
74
+ end
75
+
76
+ # Rails types a streamed template only when the response commits, as
77
+ # text/html like any response still untyped. An HTML request answered
78
+ # with content and no type yet is such a page.
79
+ def html?(payload, response)
80
+ type = response.media_type
81
+ return type == "text/html" if type
82
+ payload[:format] == :html && !NO_CONTENT.include?(payload[:status].to_i)
83
+ end
84
+
85
+ # The cookie ID the request boundary put in context, unless the response
86
+ # cannot take the cookie and the visitor brought none: a fresh ID there
87
+ # could never be set, so that page view counts by the daily hash.
88
+ def cookie_visitor?(request, settable, config)
89
+ return false unless config.visitor_cookie
90
+ return false if !settable && !request.get_header(Visitor::ENV_COOKIE)
91
+ !Context.current[:visitor_id].to_s.empty?
92
+ end
93
+
94
+ # A hook that raises has not said the page view counts.
95
+ def ignored?(config, request)
96
+ hook = config.ignore_page_view
97
+ hook ? hook.call(request) : false
98
+ rescue StandardError
99
+ true
100
+ end
101
+
102
+ # Not a response a shared cache may keep, nor one already on its way:
103
+ # ActionController::Live sends the headers at the first write or render,
104
+ # before the action ends.
105
+ def settable?(response)
106
+ return false if response.respond_to?(:committed?) && response.committed?
107
+ !public?(response)
108
+ end
109
+
110
+ def public?(response)
111
+ control = response.cache_control if response.respond_to?(:cache_control)
112
+ return true if control.is_a?(Hash) && control[:public]
113
+ response.get_header("Cache-Control").to_s.match?(Visitor::PUBLIC)
114
+ end
115
+
116
+ def properties(request)
117
+ values = {}
118
+ route = route(request)
119
+ values["route"] = route if route
120
+ values["path"] = request.filtered_path.to_s[0, PATH_LIMIT]
121
+ values["host"] = request.host.to_s.downcase[0, HOST_LIMIT]
122
+ referrer = referrer(request.referer)
123
+ values["referrer"] = referrer if referrer
124
+ values.merge!(tags(request))
125
+ values["user_agent"] = request.user_agent.to_s[0, USER_AGENT_LIMIT]
126
+ country = country(request)
127
+ values["country"] = country if country
128
+ values
129
+ end
130
+
131
+ # Inside a mounted engine Rails gives the engine's own pattern, so the
132
+ # mount path (the request's script name) goes in front, as in the path.
133
+ def route(request)
134
+ return unless request.respond_to?(:route_uri_pattern)
135
+ pattern = request.route_uri_pattern.to_s.delete_suffix("(.:format)")
136
+ return if pattern.empty?
137
+ pattern = "#{request.script_name}#{pattern}".chomp("/")
138
+ (pattern.empty? ? "/" : pattern)[0, ROUTE_LIMIT]
139
+ rescue StandardError
140
+ nil
141
+ end
142
+
143
+ # Scheme, host and path only: a referring URL's query and fragment stay home.
144
+ def referrer(value)
145
+ return unless value.is_a?(String) && !value.strip.empty?
146
+ uri = URI.parse(value.strip)
147
+ return unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
148
+ "#{uri.scheme.downcase}://#{uri.host.downcase}#{uri.path}"[0, REFERRER_LIMIT]
149
+ rescue URI::Error
150
+ nil
151
+ end
152
+
153
+ # A tag the app's filter_parameters would mask in the path is not sent.
154
+ def tags(request)
155
+ query = request.GET
156
+ raw = TAGS.each_with_object({}) { |key, values| values[key] = query[key] if query[key].is_a?(String) }
157
+ filtered = request.parameter_filter.filter(raw)
158
+ raw.each_with_object({}) do |(key, value), tags|
159
+ next unless filtered[key] == value
160
+ value = value.strip
161
+ tags[key] = value[0, TAG_LIMIT] unless value.empty?
162
+ end
163
+ rescue StandardError
164
+ {}
165
+ end
166
+
167
+ def country(request)
168
+ COUNTRY_HEADERS.each do |header|
169
+ code = request.get_header(header).to_s.strip.upcase
170
+ return code if code.match?(/\A[A-Z]{2}\z/) && !NO_COUNTRY.include?(code)
171
+ end
172
+ nil
173
+ end
174
+ end
175
+ end
176
+ end
@@ -2,13 +2,22 @@ module RailsMind
2
2
  class Redactor
3
3
  FILTER = /password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization|cookie|credential|session|email|phone|address|credit|card|body|params|arguments|bind|sql|db\.statement|query\.text/i
4
4
  MAX_STRING = 1024
5
+ # Scrubbing reads at most this much of a string; what it sends is still at
6
+ # most MAX_STRING. Every pass below is linear, and the cap bounds the rest.
7
+ INPUT_LIMIT = 8192
8
+ # Identity, correlation and release values are not free text: a numeric
9
+ # account ID that passes the Luhn check is not a card number.
10
+ PLAIN_FIELDS = %w[id kind occurred_at status release trace_id span_id parent_span_id request_id job_id visitor_id user_id account_id].freeze
5
11
 
6
12
  def call(value, depth = 0)
7
13
  return "[TRUNCATED]" if depth > 5
8
14
  case value
9
15
  when Hash
10
16
  value.first(50).to_h.transform_keys { |key| key.to_s[0, 100] }.each_with_object({}) do |(key, item), clean|
11
- clean[key] = key.match?(FILTER) ? "[FILTERED]" : call(item, depth + 1)
17
+ clean[key] = if key.match?(FILTER) then "[FILTERED]"
18
+ elsif depth.zero? && PLAIN_FIELDS.include?(key) && (item.is_a?(String) || item.is_a?(Symbol)) then scrub(item.to_s, free_text: false)
19
+ else call(item, depth + 1)
20
+ end
12
21
  end
13
22
  when Array then value.first(50).map { |item| call(item, depth + 1) }
14
23
  when String then scrub(value)
@@ -19,30 +28,56 @@ module RailsMind
19
28
  end
20
29
  end
21
30
 
31
+ # An authorization scheme names the credential that follows it, so the
32
+ # scheme and the credential go together. Digest and AWS signature headers
33
+ # carry several credential parts, so their whole value goes.
34
+ SCHEME = "(?:bearer|basic|token|negotiate|ntlm|bot|apikey)\\s+".freeze
35
+ WHOLE_VALUE_SCHEME = "(?:digest|aws4-hmac-sha256)\\s[^\\n]*".freeze
36
+
22
37
  # Exception messages carry what generic scrubbing cannot recognize: database
23
38
  # errors quote the row, key or input that failed (and the SQL with its
24
- # literals), and Ruby prints hashes and keyword arguments with their values.
39
+ # literals), and Ruby prints hashes and keyword arguments with their values,
40
+ # header values wrapped in arrays (Net::HTTP#to_hash) included.
25
41
  DATABASE_ERROR = /\A(?:PG|Mysql2|Trilogy|SQLite3)::|StatementInvalid|RecordNotUnique|NotNullViolation|ForeignKeyViolation|ValueTooLong|RangeError\z/
26
- SENSITIVE_VALUE = /((?:"|:)?(?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization|credential|cookie|session)\w*"?\s*(?:=>|:)\s*)(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|[^\s,}\])]+)/i
42
+ SENSITIVE_VALUE = /((?:"|:)?(?:password|passwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization|credential|cookie|session|phone|ssn)\w{0,64}"?\s*(?:=>|:)\s*(?:\[(?!FILTERED\]))?)(?:\[FILTERED\]|"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|#{WHOLE_VALUE_SCHEME}|(?:#{SCHEME})?[^\s,}\])]+)/i
43
+
44
+ # PostgreSQL connections use standard_conforming_strings, where a backslash
45
+ # does not escape the quote of a '...' literal (it does inside E'...');
46
+ # MySQL lets it escape the quote. Text with a backslash is read both ways,
47
+ # and anything either reading takes for a literal goes.
48
+ STANDARD_LITERAL = /(?<!\w)[Ee]'(?:[^'\\]|\\.|'')*'?|'(?:[^']|'')*'?/
49
+ ESCAPED_LITERAL = /'(?:[^'\\]|\\.|'')*'?/
27
50
 
28
51
  def scrub_message(error)
29
- text = error.message.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�")
52
+ text = bounded(error.message.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�"))
30
53
  text = text.gsub(SENSITIVE_VALUE, '\\1"[FILTERED]"')
31
54
  .gsub(/(Failing row contains )\([^\n]*\)/, "\\1(…)")
32
55
  .gsub(/(Key \([^)\n]*\)=)\([^\n]*\)/, "\\1(…)")
33
56
  .gsub(/(invalid input (?:syntax|value) for [^:\n]*: )"[^"\n]*"/, '\\1"…"')
34
- text = text.gsub(/'(?:[^'\\]|\\.|'')*'?/, "?") if database_error?(error)
57
+ text = mask_literals(text) if database_error?(error)
35
58
  scrub(text)
36
59
  end
37
60
 
38
- def scrub(value)
39
- value.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�")
40
- .gsub(/\b[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}\b/i, "[EMAIL]")
61
+ # Userinfo runs to the last @ before the path, so an unencoded @ in a
62
+ # password cannot leave its tail behind.
63
+ USERINFO = %r{\b([a-z][a-z0-9+.\-]{0,31}://)[^\s/]*@}i
64
+ # Plain or percent-encoded, as a GET form puts an address in a query string.
65
+ EMAIL = /\b[A-Z0-9._%+\-]{1,128}(?:@|%40)[A-Z0-9.\-]{1,253}\.[A-Z]{2,}\b/i
66
+ AUTHORIZATION = /((?:proxy-)?authorization["']?\s*(?:=>|[=:])\s*(?:\[(?!FILTERED\]))?\s*["']?)(?:\[FILTERED\]|#{WHOLE_VALUE_SCHEME}|(?:#{SCHEME})?[^\s,;&"']+)/i
67
+ KEY_VALUE = /((?:password|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization)\s*(?:=>|[=:])\s*)(?:#{SCHEME})?[^\s,;&]+/i
68
+ PROVIDER_TOKEN = /\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_\w{22,}|(?:AKIA|ASIA)[A-Z0-9]{16}\b|[sr]k_(?:live|test)_[A-Za-z0-9]{10,}|xox[abposr]-[A-Za-z0-9-]{10,}|sk-(?:proj-|ant-)?[A-Za-z0-9_-]{20,}|eyJ[\w-]{5,512}\.[\w-]{5,}\.[\w-]*)/
69
+ SSN = /\b\d{3}([- ])\d{2}\1\d{4}\b/
70
+ DIGIT_RUN = /(?<!\w)\d(?:[ -]?\d)*(?!\w)/
71
+
72
+ def scrub(value, free_text: true)
73
+ text = bounded(value.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�"))
74
+ .gsub(USERINFO, '\\1[FILTERED]@')
75
+ .gsub(EMAIL, "[EMAIL]")
76
+ .gsub(AUTHORIZATION, '\\1[FILTERED]')
41
77
  .gsub(/\bBearer\s+\S+/i, "Bearer [FILTERED]")
42
- .gsub(/((?:password|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|authorization)\s*[=:]\s*)[^\s,;&]+/i, '\\1[FILTERED]')
43
- .gsub(%r{(https?://)[^\s/@]+:[^\s/@]+@}, '\\1[FILTERED]@')
44
- .gsub(%r{(https?://[^\s?#]+)[?#][^\s]*}, '\\1')
45
- .slice(0, MAX_STRING)
78
+ .gsub(KEY_VALUE, '\\1[FILTERED]')
79
+ text = text.gsub(PROVIDER_TOKEN, "[FILTERED]").gsub(SSN, "[FILTERED]").gsub(DIGIT_RUN) { |run| mask_cards(run) } if free_text
80
+ text.gsub(%r{https?://\S+}) { |url| url[/\A[^?#]*/] }.slice(0, MAX_STRING)
46
81
  end
47
82
 
48
83
  private
@@ -50,5 +85,85 @@ module RailsMind
50
85
  def database_error?(error)
51
86
  error.class.ancestors.any? { |ancestor| ancestor.name.to_s.match?(DATABASE_ERROR) }
52
87
  end
88
+
89
+ # A cut string ends at its last whitespace, so no half token is left that a
90
+ # pass could no longer recognize.
91
+ def bounded(text)
92
+ return text if text.length <= INPUT_LIMIT
93
+ text = text[0, INPUT_LIMIT]
94
+ cut = text.rindex(/\s/)
95
+ cut ? text[0, cut] : text
96
+ end
97
+
98
+ def mask_literals(text)
99
+ return text.gsub(ESCAPED_LITERAL, "?") unless text.include?("\\")
100
+ spans = [ STANDARD_LITERAL, ESCAPED_LITERAL ].flat_map { |pattern| text.to_enum(:scan, pattern).map { Regexp.last_match.offset(0) } }
101
+ output, position = +"", 0
102
+ merge(spans).each do |start, finish|
103
+ output << text[position...start] << "?"
104
+ position = finish
105
+ end
106
+ output << text[position..]
107
+ end
108
+
109
+ def merge(spans)
110
+ spans.sort.each_with_object([]) do |(start, finish), merged|
111
+ if merged.any? && start < merged.last[1]
112
+ merged.last[1] = [ merged.last[1], finish ].max
113
+ else
114
+ merged << [ start, finish ]
115
+ end
116
+ end
117
+ end
118
+
119
+ # A card number is 13 to 19 digits, maybe in groups split by spaces or
120
+ # dashes, that pass the Luhn check. A run of digit groups can hold one next
121
+ # to other numbers (an expiry, a quantity), so every span of whole groups
122
+ # with at most 19 digits is tried, and the longest that passes goes.
123
+ def mask_cards(run)
124
+ groups = run.to_enum(:scan, /\d+/).map { Regexp.last_match.offset(0) }
125
+ digits = run.delete(" -")
126
+ return run if digits.length < 13
127
+ bounds, count = [], 0
128
+ groups.each { |start, finish| bounds << [ count, count += finish - start ] }
129
+ plain, doubled = luhn_sums(digits)
130
+ output, position, first = +"", 0, 0
131
+ while first < groups.size
132
+ longest, from = nil, bounds[first][0]
133
+ (first...groups.size).each do |last|
134
+ to = bounds[last][1]
135
+ break if to - from > 19
136
+ next if to - from < 13
137
+ parity = to % 2
138
+ sum = doubled[parity][to] - doubled[parity][from] + plain[1 - parity][to] - plain[1 - parity][from]
139
+ longest = last if (sum % 10).zero?
140
+ end
141
+ if longest
142
+ output << run[position...groups[first][0]] << "[FILTERED]"
143
+ position, first = groups[longest][1], longest + 1
144
+ else
145
+ first += 1
146
+ end
147
+ end
148
+ output << run[position..]
149
+ end
150
+
151
+ # Prefix sums of each digit, plain and Luhn-doubled, split by position
152
+ # parity. In a span ending at index `to`, Luhn doubles the digits whose
153
+ # position has the parity of `to`, so any span's checksum is four lookups.
154
+ def luhn_sums(digits)
155
+ plain, doubled = Array.new(2) { [ 0 ] }, Array.new(2) { [ 0 ] }
156
+ digits.each_char.with_index do |char, index|
157
+ value = char.ord - 48
158
+ twice = value * 2
159
+ twice -= 9 if twice > 9
160
+ [ 0, 1 ].each do |parity|
161
+ own = index % 2 == parity
162
+ plain[parity] << plain[parity].last + (own ? value : 0)
163
+ doubled[parity] << doubled[parity].last + (own ? twice : 0)
164
+ end
165
+ end
166
+ [ plain, doubled ]
167
+ end
53
168
  end
54
169
  end
@@ -18,6 +18,8 @@ module RailsMind
18
18
  def send_batch(json)
19
19
  request = Net::HTTP::Post.new(@uri)
20
20
  request.body = json
21
+ zone = time_zone
22
+ request["X-RailsMind-Time-Zone"] = zone if zone
21
23
  perform(request)
22
24
  end
23
25
 
@@ -29,6 +31,18 @@ module RailsMind
29
31
 
30
32
  private
31
33
 
34
+ # The app's IANA zone (Rails' default "UTC" is Etc/UTC), so RailsMind counts
35
+ # days where the app's people live. Unknown until Rails has set it.
36
+ def time_zone
37
+ @time_zone ||= begin
38
+ zone = Time.zone_default if Time.respond_to?(:zone_default)
39
+ name = zone&.tzinfo&.name
40
+ name if name.is_a?(String) && !name.empty?
41
+ rescue StandardError
42
+ nil
43
+ end
44
+ end
45
+
32
46
  def perform(request)
33
47
  Context.suppress do
34
48
  request["Authorization"] = "Bearer #{@config.token}"
@@ -1,3 +1,3 @@
1
1
  module RailsMind
2
- VERSION = "0.3.1"
2
+ VERSION = "0.4.0"
3
3
  end