solid_observer 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +25 -10
- data/app/controllers/solid_observer/dashboard_controller.rb +20 -6
- data/app/controllers/solid_observer/traces_controller.rb +17 -0
- data/app/helpers/solid_observer/application_helper.rb +31 -3
- data/app/views/layouts/solid_observer/application.html.erb +16 -1
- data/app/views/solid_observer/cable_dashboard/_recent_events.html.erb +6 -1
- data/app/views/solid_observer/cache_dashboard/_recent_events.html.erb +6 -1
- data/app/views/solid_observer/dashboard/_component_cards.html.erb +18 -0
- data/app/views/solid_observer/dashboard/_health.html.erb +6 -0
- data/app/views/solid_observer/dashboard/_home.html.erb +10 -0
- data/app/views/solid_observer/dashboard/_queue.html.erb +138 -0
- data/app/views/solid_observer/dashboard/_unified_feed.html.erb +38 -0
- data/app/views/solid_observer/dashboard/index.html.erb +5 -138
- data/app/views/solid_observer/events/index.html.erb +6 -1
- data/app/views/solid_observer/traces/show.html.erb +80 -0
- data/config/routes.rb +1 -0
- data/db/migrate/20260630000001_add_correlation_id_to_cache_events.rb +8 -0
- data/db/migrate/20260630000002_add_correlation_id_to_cable_events.rb +8 -0
- data/lib/generators/solid_observer/templates/initializer.rb.tt +6 -2
- data/lib/solid_observer/cli/health.rb +49 -0
- data/lib/solid_observer/cli/trace.rb +106 -0
- data/lib/solid_observer/configuration.rb +1 -4
- data/lib/solid_observer/correlated.rb +23 -0
- data/lib/solid_observer/queries/trace_query.rb +151 -0
- data/lib/solid_observer/services/health_score.rb +85 -0
- data/lib/solid_observer/services/record_cable_event.rb +1 -0
- data/lib/solid_observer/services/record_cache_event.rb +1 -0
- data/lib/solid_observer/services/unified_feed.rb +101 -0
- data/lib/solid_observer/version.rb +1 -1
- data/lib/solid_observer.rb +6 -0
- data/lib/tasks/solid_observer.rake +15 -0
- metadata +17 -2
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/core_ext/numeric/time"
|
|
4
|
+
|
|
5
|
+
module SolidObserver
|
|
6
|
+
module Services
|
|
7
|
+
class HealthScore
|
|
8
|
+
STATUS_RANK = {stable: 0, degraded: 1, critical: 2}.freeze
|
|
9
|
+
UNAVAILABLE = {status: :degraded, available: false}.freeze
|
|
10
|
+
WINDOW = 15.minutes
|
|
11
|
+
|
|
12
|
+
def self.call
|
|
13
|
+
new.call
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call
|
|
17
|
+
components = collect_components
|
|
18
|
+
{
|
|
19
|
+
overall: overall_status(components.values.map { |entry| entry[:status] }),
|
|
20
|
+
components: components
|
|
21
|
+
}
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
# :reek:FeatureEnvy
|
|
27
|
+
# :reek:TooManyStatements
|
|
28
|
+
def collect_components
|
|
29
|
+
config = SolidObserver.config
|
|
30
|
+
components = {}
|
|
31
|
+
|
|
32
|
+
components[:queue] = queue_component if config.solid_queue_enabled?
|
|
33
|
+
|
|
34
|
+
unless config.realtime_mode?
|
|
35
|
+
components[:cache] = stats_component(:cache, CacheStats) if config.solid_cache_enabled?
|
|
36
|
+
components[:cable] = stats_component(:cable, CableStats) if config.solid_cable_enabled?
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
components
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def queue_component
|
|
43
|
+
snapshot = SolidObserver::QueueStats.snapshot
|
|
44
|
+
return UNAVAILABLE.dup unless snapshot[:available]
|
|
45
|
+
|
|
46
|
+
{status: queue_stability(snapshot), available: true}
|
|
47
|
+
rescue => error
|
|
48
|
+
degrade(:queue, error)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def queue_stability(snapshot)
|
|
52
|
+
return :critical if snapshot[:failed_last_hour].to_i.positive?
|
|
53
|
+
return :degraded if snapshot[:failed_last_24h].to_i.positive?
|
|
54
|
+
|
|
55
|
+
:stable
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def stats_component(name, stats_class)
|
|
59
|
+
stability = stats_class.call(window: WINDOW)[:stability]
|
|
60
|
+
return UNAVAILABLE.dup unless stability&.[](:available)
|
|
61
|
+
|
|
62
|
+
{status: stability[:state] || :degraded, available: true}
|
|
63
|
+
rescue => error
|
|
64
|
+
degrade(name, error)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def overall_status(statuses)
|
|
68
|
+
return :stable if statuses.empty?
|
|
69
|
+
|
|
70
|
+
statuses.max_by { |status| STATUS_RANK.fetch(status, 1) }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def degrade(name, error)
|
|
74
|
+
log_component_error(name, error)
|
|
75
|
+
UNAVAILABLE.dup
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def log_component_error(name, error)
|
|
79
|
+
return unless defined?(Rails)
|
|
80
|
+
|
|
81
|
+
Rails.logger&.warn("HealthScore #{name} error: #{error.message}")
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -52,6 +52,7 @@ module SolidObserver
|
|
|
52
52
|
def build_event_data
|
|
53
53
|
{
|
|
54
54
|
event_type: @event.name.delete_suffix(".action_cable"),
|
|
55
|
+
correlation_id: CorrelationIdResolver.resolve(@event),
|
|
55
56
|
channel_class: channel_class,
|
|
56
57
|
broadcasting_digest: broadcasting_digest,
|
|
57
58
|
duration: duration_in_seconds,
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidObserver
|
|
4
|
+
module Services
|
|
5
|
+
class UnifiedFeed
|
|
6
|
+
SAFE_KEYS = %i[component event_type recorded_at detail correlation_id error_class].freeze
|
|
7
|
+
|
|
8
|
+
def self.call(limit: 20)
|
|
9
|
+
new(limit: limit).call
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def initialize(limit:)
|
|
13
|
+
@limit = limit
|
|
14
|
+
@config = SolidObserver.config
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def call
|
|
18
|
+
return [] if @config.realtime_mode?
|
|
19
|
+
|
|
20
|
+
rows = []
|
|
21
|
+
rows.concat(queue_rows) if @config.solid_queue_enabled?
|
|
22
|
+
rows.concat(cache_rows) if @config.solid_cache_enabled?
|
|
23
|
+
rows.concat(cable_rows) if @config.solid_cable_enabled?
|
|
24
|
+
|
|
25
|
+
rows.sort_by { |row| row[:recorded_at] || Time.at(0) }.reverse.first(@limit)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def queue_rows
|
|
31
|
+
QueueEvent.recent(@limit).map { |e| normalize_queue(e) }
|
|
32
|
+
rescue => error
|
|
33
|
+
log_error(:queue, error)
|
|
34
|
+
[]
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def cache_rows
|
|
38
|
+
CacheEvent.recent(@limit).map { |e| normalize_cache(e) }
|
|
39
|
+
rescue => error
|
|
40
|
+
log_error(:cache, error)
|
|
41
|
+
[]
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def cable_rows
|
|
45
|
+
CableEvent.recent(@limit).map { |e| normalize_cable(e) }
|
|
46
|
+
rescue => error
|
|
47
|
+
log_error(:cable, error)
|
|
48
|
+
[]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def normalize_queue(event)
|
|
52
|
+
{
|
|
53
|
+
component: :queue,
|
|
54
|
+
event_type: event.event_type,
|
|
55
|
+
recorded_at: event.recorded_at,
|
|
56
|
+
detail: [event.job_class, event.queue_name].compact.join(" · "),
|
|
57
|
+
correlation_id: event.correlation_id,
|
|
58
|
+
error_class: nil # queue events have no error_class column
|
|
59
|
+
}
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def normalize_cache(event)
|
|
63
|
+
{
|
|
64
|
+
component: :cache,
|
|
65
|
+
event_type: event.event_type,
|
|
66
|
+
recorded_at: event.recorded_at,
|
|
67
|
+
detail: cache_detail(event),
|
|
68
|
+
correlation_id: event.correlation_id,
|
|
69
|
+
error_class: event.error_class
|
|
70
|
+
}
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def normalize_cable(event)
|
|
74
|
+
{
|
|
75
|
+
component: :cable,
|
|
76
|
+
event_type: event.event_type,
|
|
77
|
+
recorded_at: event.recorded_at,
|
|
78
|
+
detail: cable_detail(event),
|
|
79
|
+
correlation_id: event.correlation_id,
|
|
80
|
+
error_class: event.error_class
|
|
81
|
+
}
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def cache_detail(event)
|
|
85
|
+
digest = event.key_digest.to_s
|
|
86
|
+
digest = "#{digest.first(10)}…" if digest.length > 10
|
|
87
|
+
digest
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def cable_detail(event)
|
|
91
|
+
event.channel_class.presence || event.broadcasting_digest.to_s.first(10)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def log_error(component, error)
|
|
95
|
+
return unless defined?(Rails) && Rails.logger
|
|
96
|
+
|
|
97
|
+
Rails.logger.warn("UnifiedFeed #{component} error: #{error.message}")
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
data/lib/solid_observer.rb
CHANGED
|
@@ -5,6 +5,7 @@ require "active_support/lazy_load_hooks"
|
|
|
5
5
|
require_relative "solid_observer/version"
|
|
6
6
|
require_relative "solid_observer/configuration"
|
|
7
7
|
require_relative "solid_observer/correlation_id_resolver"
|
|
8
|
+
require_relative "solid_observer/correlated"
|
|
8
9
|
require_relative "solid_observer/params/jobs_filter"
|
|
9
10
|
require_relative "solid_observer/params/events_filter"
|
|
10
11
|
require_relative "solid_observer/services/ui_auth_check"
|
|
@@ -13,6 +14,8 @@ require_relative "solid_observer/cli/base"
|
|
|
13
14
|
require_relative "solid_observer/cli/status"
|
|
14
15
|
require_relative "solid_observer/cli/storage"
|
|
15
16
|
require_relative "solid_observer/cli/jobs"
|
|
17
|
+
require_relative "solid_observer/cli/trace"
|
|
18
|
+
require_relative "solid_observer/cli/health"
|
|
16
19
|
require_relative "solid_observer/queue_stats"
|
|
17
20
|
require_relative "solid_observer/engine" if defined?(Rails::Engine)
|
|
18
21
|
|
|
@@ -22,6 +25,7 @@ ActiveSupport.on_load(:active_record) do
|
|
|
22
25
|
require_relative "solid_observer/base_metric"
|
|
23
26
|
require_relative "solid_observer/queries/job_executions_query"
|
|
24
27
|
require_relative "solid_observer/queries/events_query"
|
|
28
|
+
require_relative "solid_observer/queries/trace_query"
|
|
25
29
|
require_relative "solid_observer/queries/execution_finder"
|
|
26
30
|
require_relative "solid_observer/services/record_event"
|
|
27
31
|
require_relative "solid_observer/services/flush_event_buffer"
|
|
@@ -45,6 +49,8 @@ ActiveSupport.on_load(:active_record) do
|
|
|
45
49
|
require_relative "solid_observer/services/flush_cable_event_buffer"
|
|
46
50
|
require_relative "solid_observer/services/cable_stats"
|
|
47
51
|
require_relative "solid_observer/services/cable_operations"
|
|
52
|
+
require_relative "solid_observer/services/health_score"
|
|
53
|
+
require_relative "solid_observer/services/unified_feed"
|
|
48
54
|
end
|
|
49
55
|
|
|
50
56
|
module SolidObserver
|
|
@@ -204,11 +204,26 @@ namespace :solid_observer do
|
|
|
204
204
|
SolidObserver::CLI::Status.call
|
|
205
205
|
end
|
|
206
206
|
|
|
207
|
+
desc "Display aggregate and per-component health status"
|
|
208
|
+
task health: :environment do
|
|
209
|
+
SolidObserver::CLI::Health.call
|
|
210
|
+
end
|
|
211
|
+
|
|
207
212
|
desc "Display storage information and database statistics"
|
|
208
213
|
task storage: :environment do
|
|
209
214
|
SolidObserver::CLI::Storage.call
|
|
210
215
|
end
|
|
211
216
|
|
|
217
|
+
desc "Show the cross-component trace for a correlation_id"
|
|
218
|
+
task :trace, [:correlation_id, :limit] => :environment do |_t, args|
|
|
219
|
+
if args[:correlation_id].nil?
|
|
220
|
+
puts "Error: correlation_id required. Usage: rails solid_observer:trace[CORRELATION_ID,LIMIT]"
|
|
221
|
+
exit 1
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
SolidObserver::CLI::Trace.call(correlation_id: args[:correlation_id], limit: args[:limit])
|
|
225
|
+
end
|
|
226
|
+
|
|
212
227
|
namespace :jobs do
|
|
213
228
|
desc "List jobs with optional filters (status, queue, class, limit)"
|
|
214
229
|
task :list, [:status, :queue, :job_class, :limit] => :environment do |_t, args|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: solid_observer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- BartOz
|
|
@@ -62,6 +62,7 @@ files:
|
|
|
62
62
|
- app/controllers/solid_observer/events_controller.rb
|
|
63
63
|
- app/controllers/solid_observer/jobs_controller.rb
|
|
64
64
|
- app/controllers/solid_observer/storages_controller.rb
|
|
65
|
+
- app/controllers/solid_observer/traces_controller.rb
|
|
65
66
|
- app/helpers/solid_observer/application_helper.rb
|
|
66
67
|
- app/helpers/solid_observer/dashboard_helper.rb
|
|
67
68
|
- app/jobs/solid_observer/cleanup_job.rb
|
|
@@ -85,10 +86,15 @@ files:
|
|
|
85
86
|
- app/views/solid_observer/cache_operations/_confirm_clear.html.erb
|
|
86
87
|
- app/views/solid_observer/cache_operations/index.html.erb
|
|
87
88
|
- app/views/solid_observer/dashboard/_chart.html.erb
|
|
89
|
+
- app/views/solid_observer/dashboard/_component_cards.html.erb
|
|
90
|
+
- app/views/solid_observer/dashboard/_health.html.erb
|
|
91
|
+
- app/views/solid_observer/dashboard/_home.html.erb
|
|
88
92
|
- app/views/solid_observer/dashboard/_live_state.html.erb
|
|
93
|
+
- app/views/solid_observer/dashboard/_queue.html.erb
|
|
89
94
|
- app/views/solid_observer/dashboard/_queue_table.html.erb
|
|
90
95
|
- app/views/solid_observer/dashboard/_right_now.html.erb
|
|
91
96
|
- app/views/solid_observer/dashboard/_throughput.html.erb
|
|
97
|
+
- app/views/solid_observer/dashboard/_unified_feed.html.erb
|
|
92
98
|
- app/views/solid_observer/dashboard/index.html.erb
|
|
93
99
|
- app/views/solid_observer/errors/storage_unavailable.html.erb
|
|
94
100
|
- app/views/solid_observer/events/index.html.erb
|
|
@@ -99,6 +105,7 @@ files:
|
|
|
99
105
|
- app/views/solid_observer/shared/_pagination.html.erb
|
|
100
106
|
- app/views/solid_observer/shared/_stat_card.html.erb
|
|
101
107
|
- app/views/solid_observer/storages/show.html.erb
|
|
108
|
+
- app/views/solid_observer/traces/show.html.erb
|
|
102
109
|
- config/routes.rb
|
|
103
110
|
- db/migrate/20260115000001_create_solid_observer_queue_events.rb
|
|
104
111
|
- db/migrate/20260115000002_create_solid_observer_metrics.rb
|
|
@@ -110,6 +117,8 @@ files:
|
|
|
110
117
|
- db/migrate/20260612000001_add_event_type_recorded_at_index_to_cache_events.rb
|
|
111
118
|
- db/migrate/20260619000001_create_solid_observer_cable_events.rb
|
|
112
119
|
- db/migrate/20260619000002_create_solid_observer_cable_metrics.rb
|
|
120
|
+
- db/migrate/20260630000001_add_correlation_id_to_cache_events.rb
|
|
121
|
+
- db/migrate/20260630000002_add_correlation_id_to_cable_events.rb
|
|
113
122
|
- lib/generators/solid_observer/install_generator.rb
|
|
114
123
|
- lib/generators/solid_observer/templates/initializer.rb.tt
|
|
115
124
|
- lib/solid_observer.rb
|
|
@@ -124,10 +133,13 @@ files:
|
|
|
124
133
|
- lib/solid_observer/cache_subscriber.rb
|
|
125
134
|
- lib/solid_observer/chart_buffer.rb
|
|
126
135
|
- lib/solid_observer/cli/base.rb
|
|
136
|
+
- lib/solid_observer/cli/health.rb
|
|
127
137
|
- lib/solid_observer/cli/jobs.rb
|
|
128
138
|
- lib/solid_observer/cli/status.rb
|
|
129
139
|
- lib/solid_observer/cli/storage.rb
|
|
140
|
+
- lib/solid_observer/cli/trace.rb
|
|
130
141
|
- lib/solid_observer/configuration.rb
|
|
142
|
+
- lib/solid_observer/correlated.rb
|
|
131
143
|
- lib/solid_observer/correlation_id_resolver.rb
|
|
132
144
|
- lib/solid_observer/engine.rb
|
|
133
145
|
- lib/solid_observer/event_buffer_core.rb
|
|
@@ -136,6 +148,7 @@ files:
|
|
|
136
148
|
- lib/solid_observer/queries/events_query.rb
|
|
137
149
|
- lib/solid_observer/queries/execution_finder.rb
|
|
138
150
|
- lib/solid_observer/queries/job_executions_query.rb
|
|
151
|
+
- lib/solid_observer/queries/trace_query.rb
|
|
139
152
|
- lib/solid_observer/queue_event_buffer.rb
|
|
140
153
|
- lib/solid_observer/queue_stats.rb
|
|
141
154
|
- lib/solid_observer/services/cable_operations.rb
|
|
@@ -149,6 +162,7 @@ files:
|
|
|
149
162
|
- lib/solid_observer/services/flush_cache_event_buffer.rb
|
|
150
163
|
- lib/solid_observer/services/flush_cache_metrics.rb
|
|
151
164
|
- lib/solid_observer/services/flush_event_buffer.rb
|
|
165
|
+
- lib/solid_observer/services/health_score.rb
|
|
152
166
|
- lib/solid_observer/services/install_migrations.rb
|
|
153
167
|
- lib/solid_observer/services/record_cable_event.rb
|
|
154
168
|
- lib/solid_observer/services/record_cable_metric.rb
|
|
@@ -157,6 +171,7 @@ files:
|
|
|
157
171
|
- lib/solid_observer/services/record_event.rb
|
|
158
172
|
- lib/solid_observer/services/storage_info_snapshot.rb
|
|
159
173
|
- lib/solid_observer/services/ui_auth_check.rb
|
|
174
|
+
- lib/solid_observer/services/unified_feed.rb
|
|
160
175
|
- lib/solid_observer/subscriber.rb
|
|
161
176
|
- lib/solid_observer/version.rb
|
|
162
177
|
- lib/tasks/solid_observer.rake
|
|
@@ -183,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
183
198
|
- !ruby/object:Gem::Version
|
|
184
199
|
version: '0'
|
|
185
200
|
requirements: []
|
|
186
|
-
rubygems_version:
|
|
201
|
+
rubygems_version: 3.6.9
|
|
187
202
|
specification_version: 4
|
|
188
203
|
summary: Observability for the Rails 8's Solid Stack
|
|
189
204
|
test_files: []
|