rails_error_dashboard 0.11.5 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -2
- data/app/jobs/rails_error_dashboard/async_error_logging_job.rb +31 -3
- data/app/jobs/rails_error_dashboard/baseline_calculation_job.rb +21 -0
- data/app/models/rails_error_dashboard/error_log.rb +4 -7
- data/app/models/rails_error_dashboard/error_logs_record.rb +34 -0
- data/db/migrate/20260908000001_add_release_to_error_occurrences.rb +45 -0
- data/lib/rails_error_dashboard/commands/find_or_increment_error.rb +20 -9
- data/lib/rails_error_dashboard/commands/flush_storm_counts.rb +63 -28
- data/lib/rails_error_dashboard/commands/log_error.rb +11 -2
- data/lib/rails_error_dashboard/configuration.rb +7 -0
- data/lib/rails_error_dashboard/queries/analytics_stats.rb +4 -1
- data/lib/rails_error_dashboard/queries/baseline_stats.rb +39 -4
- data/lib/rails_error_dashboard/queries/dashboard_stats.rb +5 -16
- data/lib/rails_error_dashboard/queries/release_timeline.rb +72 -1
- data/lib/rails_error_dashboard/services/baseline_calculator.rb +75 -62
- data/lib/rails_error_dashboard/services/error_hash_generator.rb +9 -0
- data/lib/rails_error_dashboard/services/local_variable_capturer.rb +33 -3
- data/lib/rails_error_dashboard/services/storm_protection/count_buffer.rb +83 -22
- data/lib/rails_error_dashboard/services/storm_protection/gate.rb +52 -12
- data/lib/rails_error_dashboard/services/system_health_snapshot.rb +61 -2
- data/lib/rails_error_dashboard/value_objects/error_context.rb +19 -2
- data/lib/rails_error_dashboard/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 31b48d634cb3ef3737bfc4744993cd315deec8555bc6a0a7da55d6a3e3b011d5
|
|
4
|
+
data.tar.gz: 32fbeecb52dcbb0b48bf3f10f9fcc6da854ac20e0f1236a8fe0d37d1a3288475
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 76346c5b3c5b5ba64e272d27024dc29a4635eca2024458a10b1eff9b0a842dd5cb96bbd671566d59ce3b53f44d5ca7d61250f4f620f6b4e0e4212f09d2c884d4
|
|
7
|
+
data.tar.gz: e9fcc3afb33b3d550c830453e1d8e1325081e6f182103975855dc337c56ea1735fbfccc3879b8af35cb1fdb119eb925ccff5b516108a2eb9d3eb300ff5094e8f
|
data/README.md
CHANGED
|
@@ -184,7 +184,7 @@ config.enable_breadcrumbs = true
|
|
|
184
184
|
|
|
185
185
|
Know your app's runtime state at the moment of failure — GC stats, process memory, thread count, connection pool utilization, Puma thread stats, RubyVM cache health, YJIT compilation stats, and deep runtime insights captured automatically.
|
|
186
186
|
|
|
187
|
-
- Sub-millisecond
|
|
187
|
+
- Sub-millisecond for the in-process metrics, every metric individually rescue-wrapped. The one exception is job-queue depth (five `COUNT`s for Solid Queue, a Redis round-trip for Sidekiq): those are queries against your queue store, cached for 10 s per process and switchable off with `config.system_health_queue_stats = false`
|
|
188
188
|
- No ObjectSpace scanning, no Thread backtraces, no subprocess calls
|
|
189
189
|
- RubyVM.stat: constant cache invalidations, shape cache stats
|
|
190
190
|
- YJIT runtime stats: compiled iseqs, invalidation count, code region sizes
|
|
@@ -476,9 +476,10 @@ Seven analysis engines built in:
|
|
|
476
476
|
<details>
|
|
477
477
|
<summary><strong>Local Variable + Instance Variable Capture</strong></summary>
|
|
478
478
|
|
|
479
|
-
See the
|
|
479
|
+
See the values of local variables and instance variables at the moment an exception was raised — the most valuable debugging context possible.
|
|
480
480
|
|
|
481
481
|
- TracePoint(`:raise`) captures locals and ivars before the stack unwinds
|
|
482
|
+
- Strings, arrays and hashes are snapshotted at raise time (one level deep, bounded by the limits below), so an `ensure` block that cleans up state does not overwrite what you see. Other objects are kept by reference and show their state at serialization time
|
|
482
483
|
- Configurable limits: max variable count, nesting depth, string truncation length
|
|
483
484
|
- Sensitive data auto-filtered via Rails `filter_parameters` — passwords, tokens, and PII never stored
|
|
484
485
|
- Never stores Binding objects — values extracted immediately, Binding is GC'd
|
|
@@ -37,8 +37,16 @@ module RailsErrorDashboard
|
|
|
37
37
|
# Reconstruct exception from serialized data
|
|
38
38
|
# @param data [Hash] Serialized exception data
|
|
39
39
|
# @return [Exception] Reconstructed exception object
|
|
40
|
+
#
|
|
41
|
+
# Never relies on the subclass constructor: many real exception classes
|
|
42
|
+
# take something other than a message (ActiveRecord::RecordInvalid wants
|
|
43
|
+
# the record, custom errors take keywords), and `Klass.new(message)` on
|
|
44
|
+
# those raised inside the job, which rescued it and dropped the capture.
|
|
45
|
+
# Allocate the right class so error_type/fingerprint stay correct, then
|
|
46
|
+
# set the message with Exception's own initializer, bypassing whatever
|
|
47
|
+
# the subclass expects. Fall back to the constructor for classes whose
|
|
48
|
+
# `message` needs state their initializer sets.
|
|
40
49
|
def reconstruct_exception(data)
|
|
41
|
-
# Get or create the exception class
|
|
42
50
|
exception_class = begin
|
|
43
51
|
data[:class_name].constantize
|
|
44
52
|
rescue NameError
|
|
@@ -46,13 +54,33 @@ module RailsErrorDashboard
|
|
|
46
54
|
StandardError
|
|
47
55
|
end
|
|
48
56
|
|
|
49
|
-
|
|
50
|
-
|
|
57
|
+
exception = allocate_exception(exception_class, data[:message]) ||
|
|
58
|
+
construct_exception(exception_class, data[:message]) ||
|
|
59
|
+
StandardError.new(data[:message])
|
|
51
60
|
|
|
52
61
|
# Restore the backtrace
|
|
53
62
|
exception.set_backtrace(data[:backtrace]) if data[:backtrace]
|
|
54
63
|
|
|
55
64
|
exception
|
|
56
65
|
end
|
|
66
|
+
|
|
67
|
+
def allocate_exception(exception_class, message)
|
|
68
|
+
return nil unless exception_class < Exception
|
|
69
|
+
|
|
70
|
+
exception = exception_class.allocate
|
|
71
|
+
Exception.instance_method(:initialize).bind_call(exception, message)
|
|
72
|
+
exception.message # a subclass `message` that needs constructor state raises here
|
|
73
|
+
exception
|
|
74
|
+
rescue StandardError, NoMemoryError
|
|
75
|
+
nil
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def construct_exception(exception_class, message)
|
|
79
|
+
exception = exception_class.new(message)
|
|
80
|
+
exception.message
|
|
81
|
+
exception
|
|
82
|
+
rescue StandardError
|
|
83
|
+
nil
|
|
84
|
+
end
|
|
57
85
|
end
|
|
58
86
|
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
# Recalculates every error-type/platform baseline from recent history.
|
|
5
|
+
#
|
|
6
|
+
# Nothing in the gem schedules this: run it from the host's scheduler
|
|
7
|
+
# (Solid Queue recurring tasks, sidekiq-cron, whenever, cron + runner)
|
|
8
|
+
# roughly daily. Without it there are no baselines and no baseline alerts.
|
|
9
|
+
class BaselineCalculationJob < ApplicationJob
|
|
10
|
+
queue_as :default
|
|
11
|
+
|
|
12
|
+
def perform
|
|
13
|
+
result = Services::BaselineCalculator.calculate_all_baselines
|
|
14
|
+
Rails.logger.info("[RailsErrorDashboard] Baselines recalculated: #{result[:calculated]}")
|
|
15
|
+
result
|
|
16
|
+
rescue => e
|
|
17
|
+
Rails.logger.error("[RailsErrorDashboard] BaselineCalculationJob failed: #{e.class}: #{e.message}")
|
|
18
|
+
raise
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -339,13 +339,10 @@ module RailsErrorDashboard
|
|
|
339
339
|
return { anomaly: false, message: "Feature disabled" } unless RailsErrorDashboard.configuration.enable_baseline_alerts
|
|
340
340
|
return { anomaly: false, message: "No baseline available" } unless defined?(Queries::BaselineStats)
|
|
341
341
|
|
|
342
|
-
#
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
).where("occurred_at >= ?", Time.current.beginning_of_day).count
|
|
347
|
-
|
|
348
|
-
Queries::BaselineStats.new(error_type, platform).check_anomaly(today_count, sensitivity: sensitivity)
|
|
342
|
+
# Current hour / day / week counted in the baselines' own units — a
|
|
343
|
+
# day's count against an hourly baseline flagged everything.
|
|
344
|
+
Queries::BaselineStats.new(error_type, platform)
|
|
345
|
+
.check_current_anomaly(sensitivity: sensitivity, application_id: application_id)
|
|
349
346
|
end
|
|
350
347
|
|
|
351
348
|
# Detect cyclical occurrence patterns (daily/weekly rhythms)
|
|
@@ -34,5 +34,39 @@ module RailsErrorDashboard
|
|
|
34
34
|
# Database connection will be configured by the engine initializer
|
|
35
35
|
# after the user's configuration is loaded
|
|
36
36
|
# See lib/rails_error_dashboard/engine.rb
|
|
37
|
+
|
|
38
|
+
# Rails' default for a string column when the adapter has one (MySQL).
|
|
39
|
+
DEFAULT_STRING_LIMIT = 255
|
|
40
|
+
|
|
41
|
+
# Truncate string-typed attributes to their column limits. MySQL in strict
|
|
42
|
+
# mode rejects an over-long value outright, the capture path rescues the
|
|
43
|
+
# failure, and the error is silently lost — over a 300-character app
|
|
44
|
+
# version. Columns without a declared limit are clamped to the Rails
|
|
45
|
+
# default so behaviour is the same on every adapter. Text columns are
|
|
46
|
+
# untouched. Identity (error_hash) is computed from the raw values before
|
|
47
|
+
# this runs, so clamping display metadata never changes grouping.
|
|
48
|
+
# @param attributes [Hash] attribute name => value
|
|
49
|
+
# @return [Hash] a copy with over-long strings truncated
|
|
50
|
+
def self.clamp_string_attributes(attributes)
|
|
51
|
+
limits = string_column_limits
|
|
52
|
+
attributes.each_with_object({}) do |(name, value), clamped|
|
|
53
|
+
limit = limits[name.to_s]
|
|
54
|
+
clamped[name] = if limit && value.is_a?(String) && value.length > limit
|
|
55
|
+
value[0, limit]
|
|
56
|
+
else
|
|
57
|
+
value
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
rescue => e
|
|
61
|
+
RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] clamp_string_attributes failed: #{e.message}")
|
|
62
|
+
attributes
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# @return [Hash] column name => character limit, string columns only (memoized per class)
|
|
66
|
+
def self.string_column_limits
|
|
67
|
+
@string_column_limits ||= columns_hash.each_with_object({}) do |(name, column), limits|
|
|
68
|
+
limits[name] = column.limit || DEFAULT_STRING_LIMIT if column.type == :string
|
|
69
|
+
end
|
|
70
|
+
end
|
|
37
71
|
end
|
|
38
72
|
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Release attribution belongs on the occurrence, not the group.
|
|
4
|
+
#
|
|
5
|
+
# An ErrorLog row is a GROUP: it keeps the app_version/git_sha of the first
|
|
6
|
+
# capture in its window and recurrences do not touch them. So an error first
|
|
7
|
+
# seen in v1 that kept firing after the v2 deploy was reported entirely under
|
|
8
|
+
# v1, and the release timeline could not answer "did this deploy make things
|
|
9
|
+
# worse". Each ErrorOccurrence now records the release it happened under, and
|
|
10
|
+
# ReleaseTimeline counts occurrences per release.
|
|
11
|
+
#
|
|
12
|
+
# Existing occurrences are backfilled from their group's release. That is the
|
|
13
|
+
# same attribution the dashboard showed before this migration (no worse), and
|
|
14
|
+
# it keeps historical releases populated instead of dropping to zero.
|
|
15
|
+
class AddReleaseToErrorOccurrences < ActiveRecord::Migration[7.0]
|
|
16
|
+
def up
|
|
17
|
+
return if column_exists?(:rails_error_dashboard_error_occurrences, :app_version)
|
|
18
|
+
|
|
19
|
+
add_column :rails_error_dashboard_error_occurrences, :app_version, :string
|
|
20
|
+
add_column :rails_error_dashboard_error_occurrences, :git_sha, :string
|
|
21
|
+
add_index :rails_error_dashboard_error_occurrences, [ :app_version, :occurred_at ],
|
|
22
|
+
name: "index_error_occurrences_on_version_and_time"
|
|
23
|
+
|
|
24
|
+
# Correlated subqueries rather than UPDATE ... FROM / JOIN: the one form
|
|
25
|
+
# SQLite, PostgreSQL and MySQL all accept.
|
|
26
|
+
execute <<~SQL.squish
|
|
27
|
+
UPDATE rails_error_dashboard_error_occurrences
|
|
28
|
+
SET app_version = (
|
|
29
|
+
SELECT app_version FROM rails_error_dashboard_error_logs
|
|
30
|
+
WHERE rails_error_dashboard_error_logs.id = rails_error_dashboard_error_occurrences.error_log_id
|
|
31
|
+
),
|
|
32
|
+
git_sha = (
|
|
33
|
+
SELECT git_sha FROM rails_error_dashboard_error_logs
|
|
34
|
+
WHERE rails_error_dashboard_error_logs.id = rails_error_dashboard_error_occurrences.error_log_id
|
|
35
|
+
)
|
|
36
|
+
WHERE app_version IS NULL
|
|
37
|
+
SQL
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def down
|
|
41
|
+
remove_index :rails_error_dashboard_error_occurrences, name: "index_error_occurrences_on_version_and_time"
|
|
42
|
+
remove_column :rails_error_dashboard_error_occurrences, :git_sha
|
|
43
|
+
remove_column :rails_error_dashboard_error_occurrences, :app_version
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -4,6 +4,10 @@ module RailsErrorDashboard
|
|
|
4
4
|
module Commands
|
|
5
5
|
# Command: Find an existing error by hash or create a new one
|
|
6
6
|
# Uses pessimistic locking to prevent race conditions in multi-app scenarios.
|
|
7
|
+
# The whole find-and-write runs inside ONE transaction: a row lock taken
|
|
8
|
+
# by SELECT ... FOR UPDATE lives only as long as the transaction that took
|
|
9
|
+
# it, so a lock on a standalone SELECT was released before the UPDATE ran
|
|
10
|
+
# and two concurrent captures could both read count N and both write N+1.
|
|
7
11
|
#
|
|
8
12
|
# Search order:
|
|
9
13
|
# 1. Unresolved errors with same hash within 24 hours → increment occurrence count
|
|
@@ -37,16 +41,18 @@ module RailsErrorDashboard
|
|
|
37
41
|
end
|
|
38
42
|
|
|
39
43
|
def call
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
44
|
+
ErrorLog.transaction do
|
|
45
|
+
# Priority 1: Find unresolved match (existing behavior)
|
|
46
|
+
existing = find_unresolved
|
|
47
|
+
next increment_existing(existing) if existing
|
|
43
48
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
# Priority 2: Find resolved/wont_fix match → reopen
|
|
50
|
+
resolved = find_resolved
|
|
51
|
+
next reopen_existing(resolved) if resolved
|
|
47
52
|
|
|
48
|
-
|
|
49
|
-
|
|
53
|
+
# Priority 3: Create new record
|
|
54
|
+
create_new_or_retry
|
|
55
|
+
end
|
|
50
56
|
end
|
|
51
57
|
|
|
52
58
|
private
|
|
@@ -132,7 +138,12 @@ module RailsErrorDashboard
|
|
|
132
138
|
end
|
|
133
139
|
|
|
134
140
|
def create_new_or_retry
|
|
135
|
-
|
|
141
|
+
# Savepoint: on PostgreSQL a unique-violation poisons the enclosing
|
|
142
|
+
# transaction, and the retry lookups below would fail with "current
|
|
143
|
+
# transaction is aborted" instead of finding the winner's row.
|
|
144
|
+
ErrorLog.transaction(requires_new: true) do
|
|
145
|
+
ErrorLog.create!(@attributes.reverse_merge(resolved: false))
|
|
146
|
+
end
|
|
136
147
|
rescue ActiveRecord::RecordNotUnique
|
|
137
148
|
# Race condition: another process created the same error
|
|
138
149
|
retry_existing = with_environment(
|
|
@@ -8,7 +8,8 @@ module RailsErrorDashboard
|
|
|
8
8
|
# Runs in a background job (DB allowed). For each counted fingerprint:
|
|
9
9
|
# 1. Recompute the canonical error_hash from the stored identity parts
|
|
10
10
|
# (the gate's key deliberately omits application_id — resolved here)
|
|
11
|
-
# 2. Unresolved match →
|
|
11
|
+
# 2. Unresolved match → the ONE row FindOrIncrementError would pick
|
|
12
|
+
# (24 h window, exact environment first): occurrence_count += N
|
|
12
13
|
# 3. Resolved match → reopen (mirrors FindOrIncrementError semantics)
|
|
13
14
|
# 4. No match → create a minimal ErrorLog from the exemplar
|
|
14
15
|
#
|
|
@@ -59,26 +60,33 @@ module RailsErrorDashboard
|
|
|
59
60
|
|
|
60
61
|
error_hash = canonical_hash(entry, application)
|
|
61
62
|
last_seen = parse_time(entry["last_seen_at"]) || Time.current
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
63
|
+
# An explicit environment captured at the gate wins over the worker's
|
|
64
|
+
# own, mirroring LogError#resolve_environment for full captures.
|
|
65
|
+
env = current_environment && (entry["environment"].presence || current_environment)
|
|
66
|
+
|
|
67
|
+
# Priority 1: unresolved match — ONE row, chosen exactly as
|
|
68
|
+
# FindOrIncrementError chooses it (same hash + application, occurred
|
|
69
|
+
# within 24 h, exact environment before a legacy NULL row, most
|
|
70
|
+
# recently seen first), then a single atomic UPDATE on that id.
|
|
71
|
+
#
|
|
72
|
+
# It must be one row: the normal path opens a fresh group once the
|
|
73
|
+
# previous one's occurred_at falls outside the 24 h window, so a
|
|
74
|
+
# long-running unresolved error legitimately owns several unresolved
|
|
75
|
+
# rows. An update_all across the whole hash would add N to every one
|
|
76
|
+
# of them — seven real events becoming twelve counted occurrences.
|
|
77
|
+
target = unresolved_target(error_hash, application, env)
|
|
78
|
+
if target
|
|
79
|
+
if env && target.environment.blank?
|
|
80
|
+
ErrorLog.where(id: target.id).update_all([
|
|
81
|
+
"occurrence_count = occurrence_count + ?, last_seen_at = ?, environment = ?",
|
|
82
|
+
count, last_seen, env
|
|
83
|
+
])
|
|
84
|
+
else
|
|
85
|
+
ErrorLog.where(id: target.id).update_all([
|
|
86
|
+
"occurrence_count = occurrence_count + ?, last_seen_at = ?", count, last_seen
|
|
87
|
+
])
|
|
88
|
+
end
|
|
89
|
+
return count
|
|
82
90
|
end
|
|
83
91
|
|
|
84
92
|
# Priority 2: resolved/wont_fix match — reopen, mirroring
|
|
@@ -107,12 +115,17 @@ module RailsErrorDashboard
|
|
|
107
115
|
|
|
108
116
|
# Priority 3: first seen during count-only mode — minimal ErrorLog
|
|
109
117
|
# from the exemplar (no backtrace/context was captured; the next
|
|
110
|
-
# occurrence after the storm fills in detail via the normal path)
|
|
118
|
+
# occurrence after the storm fills in detail via the normal path).
|
|
119
|
+
#
|
|
120
|
+
# The exemplar message is RAW — the gate stores what the exception
|
|
121
|
+
# said so the canonical hash (computed above, from the raw message,
|
|
122
|
+
# exactly as the full path does) still lands on the same row. It must
|
|
123
|
+
# therefore go through the same redaction as LogError before it is
|
|
124
|
+
# persisted; storm protection and sensitive filtering are both on by
|
|
125
|
+
# default, and an incident is exactly when a password in a message
|
|
126
|
+
# must not reach the database.
|
|
111
127
|
create_attrs = {
|
|
112
|
-
environment: env
|
|
113
|
-
}.compact
|
|
114
|
-
ErrorLog.create!(
|
|
115
|
-
**create_attrs,
|
|
128
|
+
environment: env,
|
|
116
129
|
application_id: application.id,
|
|
117
130
|
error_type: entry["error_class"],
|
|
118
131
|
message: entry["message"],
|
|
@@ -124,10 +137,32 @@ module RailsErrorDashboard
|
|
|
124
137
|
occurrence_count: count,
|
|
125
138
|
error_hash: error_hash,
|
|
126
139
|
resolved: false
|
|
127
|
-
|
|
140
|
+
}.compact
|
|
141
|
+
ErrorLog.create!(**ErrorLog.clamp_string_attributes(Services::SensitiveDataFilter.filter_attributes(create_attrs)))
|
|
128
142
|
count
|
|
129
143
|
end
|
|
130
144
|
|
|
145
|
+
|
|
146
|
+
# The unresolved row the full capture path would increment right now.
|
|
147
|
+
def unresolved_target(error_hash, application, env)
|
|
148
|
+
scope = ErrorLog.unresolved
|
|
149
|
+
.where(error_hash: error_hash, application_id: application.id)
|
|
150
|
+
.where("occurred_at >= ?", 24.hours.ago)
|
|
151
|
+
if env
|
|
152
|
+
scope = scope.where(environment: [ env, nil ])
|
|
153
|
+
.order(Arel.sql("CASE WHEN environment IS NULL THEN 1 ELSE 0 END"))
|
|
154
|
+
end
|
|
155
|
+
scope.order(last_seen_at: :desc).select(:id, :environment).first
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# The redaction LogError applies to a message, for exemplars that reach
|
|
159
|
+
# the database by any other route (first-seen rows, the storm ledger).
|
|
160
|
+
def redact_message(message)
|
|
161
|
+
return message if message.blank?
|
|
162
|
+
|
|
163
|
+
Services::SensitiveDataFilter.filter_attributes({ message: message.to_s })[:message]
|
|
164
|
+
end
|
|
165
|
+
|
|
131
166
|
# Mirrors ErrorHashGenerator.call exactly: same fields, same order,
|
|
132
167
|
# same normalization — so counts land on the same ErrorLog the full
|
|
133
168
|
# capture path would have used.
|
|
@@ -196,7 +231,7 @@ module RailsErrorDashboard
|
|
|
196
231
|
existing = event.top_fingerprints_list
|
|
197
232
|
fresh = @entries.map { |e|
|
|
198
233
|
e = e.with_indifferent_access if e.respond_to?(:with_indifferent_access)
|
|
199
|
-
{ "class" => e["error_class"], "message" => e["message"].to_s[0, 120], "count" => e["count"].to_i }
|
|
234
|
+
{ "class" => e["error_class"], "message" => redact_message(e["message"]).to_s[0, 120], "count" => e["count"].to_i }
|
|
200
235
|
}
|
|
201
236
|
|
|
202
237
|
merged = (existing + fresh)
|
|
@@ -299,6 +299,9 @@ module RailsErrorDashboard
|
|
|
299
299
|
# Apply sensitive data filtering (on by default)
|
|
300
300
|
attributes = Services::SensitiveDataFilter.filter_attributes(attributes)
|
|
301
301
|
|
|
302
|
+
# Fit string metadata to its columns (after hashing, before writing)
|
|
303
|
+
attributes = ErrorLog.clamp_string_attributes(attributes)
|
|
304
|
+
|
|
302
305
|
# Harvest breadcrumbs (if enabled and column exists)
|
|
303
306
|
if !storm_lite && ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
304
307
|
# Sync path: harvest from current thread
|
|
@@ -377,13 +380,19 @@ module RailsErrorDashboard
|
|
|
377
380
|
# Track individual error occurrence for co-occurrence analysis (if table exists)
|
|
378
381
|
if defined?(ErrorOccurrence) && ErrorOccurrence.table_exists?
|
|
379
382
|
begin
|
|
380
|
-
|
|
383
|
+
occurrence_attrs = {
|
|
381
384
|
error_log: error_log,
|
|
382
385
|
occurred_at: attributes[:occurred_at],
|
|
383
386
|
user_id: attributes[:user_id],
|
|
384
387
|
request_id: error_context.request_id,
|
|
385
388
|
session_id: error_context.session_id
|
|
386
|
-
|
|
389
|
+
}
|
|
390
|
+
# The release THIS event happened under. The group keeps its first
|
|
391
|
+
# release; per-release counts come from here (ReleaseTimeline).
|
|
392
|
+
occurrence_columns = ErrorOccurrence.column_names
|
|
393
|
+
occurrence_attrs[:app_version] = attributes[:app_version] if occurrence_columns.include?("app_version")
|
|
394
|
+
occurrence_attrs[:git_sha] = attributes[:git_sha] if occurrence_columns.include?("git_sha")
|
|
395
|
+
ErrorOccurrence.create(ErrorOccurrence.clamp_string_attributes(occurrence_attrs))
|
|
387
396
|
rescue => e
|
|
388
397
|
RailsErrorDashboard::Logger.error("Failed to create error occurrence: #{e.message}")
|
|
389
398
|
end
|
|
@@ -169,6 +169,8 @@ module RailsErrorDashboard
|
|
|
169
169
|
|
|
170
170
|
# System health snapshot (GC, memory, threads, connection pool at error time)
|
|
171
171
|
attr_accessor :enable_system_health # Master switch (default: false)
|
|
172
|
+
attr_accessor :system_health_queue_stats # Include job-queue depth counts (default: true)
|
|
173
|
+
attr_accessor :system_health_queue_stats_cache_seconds # Reuse queue counts this long per process (default: 10)
|
|
172
174
|
|
|
173
175
|
# Local variable capture via TracePoint(:raise)
|
|
174
176
|
attr_accessor :enable_local_variables # Master switch (default: false)
|
|
@@ -393,6 +395,11 @@ module RailsErrorDashboard
|
|
|
393
395
|
|
|
394
396
|
# System health snapshot defaults - OFF by default (opt-in)
|
|
395
397
|
@enable_system_health = false # Capture GC, memory, threads, connection pool at error time
|
|
398
|
+
# Queue-depth counts are queries against the queue store (five COUNTs
|
|
399
|
+
# for Solid Queue), the one part of the snapshot that is not sub-ms.
|
|
400
|
+
# Cached per process so an error burst runs them once per interval.
|
|
401
|
+
@system_health_queue_stats = true
|
|
402
|
+
@system_health_queue_stats_cache_seconds = 10
|
|
396
403
|
|
|
397
404
|
# Local variable capture defaults - OFF by default (opt-in)
|
|
398
405
|
@enable_local_variables = false # TracePoint(:raise) for local var capture
|
|
@@ -126,7 +126,10 @@ module RailsErrorDashboard
|
|
|
126
126
|
total = error_statistics[:total]
|
|
127
127
|
return 0 if total.zero?
|
|
128
128
|
|
|
129
|
-
|
|
129
|
+
# Same scoped relation as the denominator: an unscoped numerator
|
|
130
|
+
# counted every application's resolved errors against one
|
|
131
|
+
# application's total, and the "rate" went past 100%.
|
|
132
|
+
resolved_count = base_query.resolved.count
|
|
130
133
|
((resolved_count.to_f / total) * 100).round(1)
|
|
131
134
|
end
|
|
132
135
|
|
|
@@ -85,23 +85,58 @@ module RailsErrorDashboard
|
|
|
85
85
|
# @param current_count [Integer] Current error count
|
|
86
86
|
# @param sensitivity [Integer] Standard deviations threshold (default: 2)
|
|
87
87
|
# @return [Hash] { anomaly: true/false, level: Symbol, baseline_type: String }
|
|
88
|
-
|
|
89
|
-
|
|
88
|
+
# Compare a count against the first available baseline, in matching
|
|
89
|
+
# units: the current HOUR's count against the hourly baseline, TODAY's
|
|
90
|
+
# against the daily, THIS WEEK's against the weekly. A bare positional
|
|
91
|
+
# count is compared against whichever baseline is found (legacy callers).
|
|
92
|
+
#
|
|
93
|
+
# @param current_count [Integer, nil] legacy: one count used for any baseline
|
|
94
|
+
# @param hourly [Integer, nil] events so far this hour
|
|
95
|
+
# @param daily [Integer, nil] events so far today
|
|
96
|
+
# @param weekly [Integer, nil] events so far this week
|
|
97
|
+
def check_anomaly(current_count = nil, sensitivity: 2, hourly: nil, daily: nil, weekly: nil)
|
|
98
|
+
candidates = [
|
|
99
|
+
[ hourly_baseline, hourly || current_count ],
|
|
100
|
+
[ daily_baseline, daily || current_count ],
|
|
101
|
+
[ weekly_baseline, weekly || current_count ]
|
|
102
|
+
]
|
|
103
|
+
baseline, count = candidates.find { |b, c| b && c }
|
|
90
104
|
|
|
91
105
|
if baseline.nil?
|
|
92
106
|
return { anomaly: false, level: nil, baseline_type: nil, message: "No baseline available" }
|
|
93
107
|
end
|
|
94
108
|
|
|
95
|
-
level = baseline.anomaly_level(
|
|
109
|
+
level = baseline.anomaly_level(count, sensitivity: sensitivity)
|
|
96
110
|
|
|
97
111
|
{
|
|
98
112
|
anomaly: level.present?,
|
|
99
113
|
level: level,
|
|
100
114
|
baseline_type: baseline.baseline_type,
|
|
115
|
+
current_count: count,
|
|
101
116
|
threshold: baseline.threshold(sensitivity: sensitivity),
|
|
102
|
-
std_devs_above: baseline.std_devs_above_mean(
|
|
117
|
+
std_devs_above: baseline.std_devs_above_mean(count)
|
|
103
118
|
}
|
|
104
119
|
end
|
|
120
|
+
|
|
121
|
+
# Events so far in the current hour / day / week, counted in the same
|
|
122
|
+
# units the baselines were built from (Services::BaselineCalculator).
|
|
123
|
+
# @return [Hash] { hourly: Integer, daily: Integer, weekly: Integer }
|
|
124
|
+
def current_counts(application_id: nil)
|
|
125
|
+
relation = Services::BaselineCalculator.counting_relation(@error_type, @platform, application_id: application_id)
|
|
126
|
+
column = Services::BaselineCalculator.time_column
|
|
127
|
+
now = Time.current
|
|
128
|
+
{
|
|
129
|
+
hourly: relation.where("#{column} >= ?", now.beginning_of_hour).count,
|
|
130
|
+
daily: relation.where("#{column} >= ?", now.beginning_of_day).count,
|
|
131
|
+
weekly: relation.where("#{column} >= ?", now.beginning_of_week).count
|
|
132
|
+
}
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# The anomaly check for right now: current counts against their baselines.
|
|
136
|
+
# @param application_id [Integer, nil] scope the current counts to one application
|
|
137
|
+
def check_current_anomaly(sensitivity: 2, application_id: nil)
|
|
138
|
+
check_anomaly(sensitivity: sensitivity, **current_counts(application_id: application_id))
|
|
139
|
+
end
|
|
105
140
|
end
|
|
106
141
|
end
|
|
107
142
|
end
|
|
@@ -181,14 +181,8 @@ module RailsErrorDashboard
|
|
|
181
181
|
|
|
182
182
|
# Check most common error types for anomalies
|
|
183
183
|
base_scope.distinct.pluck(:error_type, :platform).compact.any? do |(error_type, platform)|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
error_type: error_type,
|
|
187
|
-
platform: platform
|
|
188
|
-
).where("occurred_at >= ?", Time.current.beginning_of_day).count
|
|
189
|
-
|
|
190
|
-
result = stats.check_anomaly(error_count, sensitivity: 2)
|
|
191
|
-
result[:anomaly]
|
|
184
|
+
Queries::BaselineStats.new(error_type, platform)
|
|
185
|
+
.check_current_anomaly(sensitivity: 2, application_id: @application_id)[:anomaly]
|
|
192
186
|
end
|
|
193
187
|
end
|
|
194
188
|
|
|
@@ -198,19 +192,14 @@ module RailsErrorDashboard
|
|
|
198
192
|
|
|
199
193
|
# Find the most anomalous error type
|
|
200
194
|
anomalies = base_scope.distinct.pluck(:error_type, :platform).compact.map do |(error_type, platform)|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
error_type: error_type,
|
|
204
|
-
platform: platform
|
|
205
|
-
).where("occurred_at >= ?", Time.current.beginning_of_day).count
|
|
206
|
-
|
|
207
|
-
result = stats.check_anomaly(error_count, sensitivity: 2)
|
|
195
|
+
result = Queries::BaselineStats.new(error_type, platform)
|
|
196
|
+
.check_current_anomaly(sensitivity: 2, application_id: @application_id)
|
|
208
197
|
next unless result[:anomaly]
|
|
209
198
|
|
|
210
199
|
{
|
|
211
200
|
error_type: error_type,
|
|
212
201
|
platform: platform,
|
|
213
|
-
count:
|
|
202
|
+
count: result[:current_count],
|
|
214
203
|
level: result[:level],
|
|
215
204
|
std_devs_above: result[:std_devs_above]
|
|
216
205
|
}
|