solid_observer 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/README.md +100 -25
- data/app/assets/javascripts/solid_observer/live_poll.js +3 -1
- data/app/controllers/solid_observer/application_controller.rb +1 -0
- data/app/controllers/solid_observer/cable_dashboard_controller.rb +52 -0
- data/app/controllers/solid_observer/cable_operations_controller.rb +16 -0
- data/app/controllers/solid_observer/cache_dashboard_controller.rb +33 -40
- data/app/controllers/solid_observer/dashboard_controller.rb +21 -13
- data/app/controllers/solid_observer/traces_controller.rb +17 -0
- data/app/helpers/solid_observer/application_helper.rb +145 -3
- data/app/models/solid_observer/cable_event.rb +13 -0
- data/app/models/solid_observer/cable_metric.rb +12 -0
- data/app/models/solid_observer/cache_metric.rb +1 -2
- data/app/models/solid_observer/storage_info.rb +1 -1
- data/app/views/layouts/solid_observer/application.html.erb +35 -9
- data/app/views/solid_observer/cable_dashboard/_charts.html.erb +31 -0
- data/app/views/solid_observer/cable_dashboard/_recent_events.html.erb +39 -0
- data/app/views/solid_observer/cable_dashboard/_summary.html.erb +34 -0
- data/app/views/solid_observer/cable_dashboard/index.html.erb +118 -0
- data/app/views/solid_observer/cache_dashboard/_recent_events.html.erb +6 -1
- data/app/views/solid_observer/dashboard/_component_cards.html.erb +18 -0
- data/app/views/solid_observer/dashboard/_health.html.erb +6 -0
- data/app/views/solid_observer/dashboard/_home.html.erb +10 -0
- data/app/views/solid_observer/dashboard/_queue.html.erb +138 -0
- data/app/views/solid_observer/dashboard/_queue_table.html.erb +1 -0
- data/app/views/solid_observer/dashboard/_unified_feed.html.erb +38 -0
- data/app/views/solid_observer/dashboard/index.html.erb +3 -139
- data/app/views/solid_observer/events/index.html.erb +7 -1
- data/app/views/solid_observer/jobs/index.html.erb +1 -0
- data/app/views/solid_observer/storages/show.html.erb +29 -3
- data/app/views/solid_observer/traces/show.html.erb +80 -0
- data/config/routes.rb +3 -0
- data/db/migrate/20260612000001_add_event_type_recorded_at_index_to_cache_events.rb +21 -0
- data/db/migrate/20260619000001_create_solid_observer_cable_events.rb +22 -0
- data/db/migrate/20260619000002_create_solid_observer_cable_metrics.rb +17 -0
- data/db/migrate/20260630000001_add_correlation_id_to_cache_events.rb +8 -0
- data/db/migrate/20260630000002_add_correlation_id_to_cable_events.rb +8 -0
- data/lib/generators/solid_observer/install_generator.rb +8 -1
- data/lib/generators/solid_observer/templates/initializer.rb.tt +24 -5
- data/lib/solid_observer/base_event.rb +1 -1
- data/lib/solid_observer/base_metric.rb +1 -1
- data/lib/solid_observer/base_record.rb +8 -0
- data/lib/solid_observer/cable_event_buffer.rb +28 -0
- data/lib/solid_observer/cable_metric_buffer.rb +230 -0
- data/lib/solid_observer/cable_subscriber.rb +57 -0
- data/lib/solid_observer/cache_event_buffer.rb +11 -36
- data/lib/solid_observer/cache_metric_buffer.rb +229 -0
- data/lib/solid_observer/chart_buffer.rb +84 -27
- data/lib/solid_observer/cli/health.rb +49 -0
- data/lib/solid_observer/cli/trace.rb +106 -0
- data/lib/solid_observer/configuration.rb +47 -7
- data/lib/solid_observer/correlated.rb +23 -0
- data/lib/solid_observer/engine.rb +46 -28
- data/lib/solid_observer/event_buffer_core.rb +218 -0
- data/lib/solid_observer/queries/trace_query.rb +151 -0
- data/lib/solid_observer/queue_event_buffer.rb +9 -201
- data/lib/solid_observer/services/cable_operations.rb +74 -0
- data/lib/solid_observer/services/cable_stats.rb +385 -0
- data/lib/solid_observer/services/cache_stats.rb +35 -18
- data/lib/solid_observer/services/cleanup_storage.rb +82 -47
- data/lib/solid_observer/services/flush_cable_event_buffer.rb +54 -0
- data/lib/solid_observer/services/flush_cable_metrics.rb +54 -0
- data/lib/solid_observer/services/flush_cache_metrics.rb +56 -0
- data/lib/solid_observer/services/health_score.rb +85 -0
- data/lib/solid_observer/services/record_cable_event.rb +115 -0
- data/lib/solid_observer/services/record_cable_metric.rb +73 -0
- data/lib/solid_observer/services/record_cache_event.rb +24 -0
- data/lib/solid_observer/services/record_cache_metric.rb +13 -21
- data/lib/solid_observer/services/storage_info_snapshot.rb +103 -15
- data/lib/solid_observer/services/unified_feed.rb +101 -0
- data/lib/solid_observer/version.rb +1 -1
- data/lib/solid_observer.rb +42 -11
- data/lib/tasks/solid_observer.rake +97 -21
- metadata +41 -6
- data/app/assets/stylesheets/solid_observer/application.css +0 -18
- data/bin/console +0 -11
- data/bin/quality_gate +0 -95
- data/bin/setup +0 -8
|
@@ -5,6 +5,57 @@ require_relative "database_size"
|
|
|
5
5
|
module SolidObserver
|
|
6
6
|
module Services
|
|
7
7
|
class StorageInfoSnapshot
|
|
8
|
+
class RecordCount
|
|
9
|
+
def initialize(connection, table_name)
|
|
10
|
+
@connection = connection
|
|
11
|
+
@table_name = table_name
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def solid_cache_count
|
|
15
|
+
case adapter_key
|
|
16
|
+
when :postgresql then postgresql_approximate_count
|
|
17
|
+
when :mysql then mysql_approximate_count
|
|
18
|
+
else
|
|
19
|
+
yield
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
attr_reader :connection, :table_name
|
|
26
|
+
|
|
27
|
+
def adapter_key
|
|
28
|
+
case connection.adapter_name.to_s.downcase
|
|
29
|
+
when /postgres|postgis/ then :postgresql
|
|
30
|
+
when "mysql2", "trilogy", "mysql" then :mysql
|
|
31
|
+
else
|
|
32
|
+
:other
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def postgresql_approximate_count
|
|
37
|
+
quoted_table = connection.quote(table_name)
|
|
38
|
+
|
|
39
|
+
connection.query_value(<<~SQL.squish)&.to_i || 0
|
|
40
|
+
SELECT COALESCE(
|
|
41
|
+
(SELECT reltuples::bigint FROM pg_class WHERE oid = to_regclass(#{quoted_table})),
|
|
42
|
+
0
|
|
43
|
+
)
|
|
44
|
+
SQL
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def mysql_approximate_count
|
|
48
|
+
quoted_table = connection.quote(table_name)
|
|
49
|
+
|
|
50
|
+
connection.query_value(<<~SQL)&.to_i || 0
|
|
51
|
+
SELECT COALESCE(table_rows, 0)
|
|
52
|
+
FROM information_schema.tables
|
|
53
|
+
WHERE table_schema = DATABASE()
|
|
54
|
+
AND table_name = #{quoted_table}
|
|
55
|
+
SQL
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
8
59
|
Component = Struct.new(:key, :label, :record_label, :model, :enabled, keyword_init: true) do
|
|
9
60
|
def enabled?
|
|
10
61
|
enabled.call
|
|
@@ -14,28 +65,25 @@ module SolidObserver
|
|
|
14
65
|
key == "solid_cache"
|
|
15
66
|
end
|
|
16
67
|
|
|
17
|
-
def
|
|
18
|
-
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
def data_source_exists?(connection, table_name)
|
|
22
|
-
connection.data_source_exists?(table_name)
|
|
68
|
+
def solid_cable?
|
|
69
|
+
key == "solid_cable"
|
|
23
70
|
end
|
|
24
71
|
|
|
25
|
-
def
|
|
26
|
-
|
|
72
|
+
def storage_model
|
|
73
|
+
model.call
|
|
27
74
|
end
|
|
28
75
|
|
|
29
76
|
def snapshot
|
|
30
77
|
return unless enabled?
|
|
31
78
|
return unavailable_snapshot(reason: "SolidCache is unavailable") if unavailable_solid_cache?
|
|
79
|
+
return unavailable_snapshot(reason: "SolidCable is unavailable") if solid_cable? && !defined?(::SolidCable::Message)
|
|
32
80
|
|
|
33
81
|
existing_data_source_snapshot
|
|
34
82
|
rescue *StorageInfoSnapshot::CONNECTION_ERRORS, TypeError
|
|
35
83
|
unavailable_snapshot(reason: "Storage unavailable")
|
|
36
84
|
end
|
|
37
85
|
|
|
38
|
-
def available_snapshot(db_size_bytes:, event_count
|
|
86
|
+
def available_snapshot(db_size_bytes:, event_count:, **extras)
|
|
39
87
|
{
|
|
40
88
|
component: key,
|
|
41
89
|
label: label,
|
|
@@ -45,7 +93,7 @@ module SolidObserver
|
|
|
45
93
|
record_label: record_label,
|
|
46
94
|
recorded_at: Time.current,
|
|
47
95
|
unavailable_reason: nil
|
|
48
|
-
}
|
|
96
|
+
}.merge(extras)
|
|
49
97
|
end
|
|
50
98
|
|
|
51
99
|
def unavailable_snapshot(reason:)
|
|
@@ -71,16 +119,41 @@ module SolidObserver
|
|
|
71
119
|
record_model = storage_model
|
|
72
120
|
connection = record_model.connection
|
|
73
121
|
table_name = record_model.table_name.to_s
|
|
74
|
-
return
|
|
122
|
+
return unavailable_snapshot(reason: "Table unavailable") unless data_source_available?(connection, table_name)
|
|
123
|
+
|
|
124
|
+
build_available_snapshot(record_model, connection, table_name)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def build_available_snapshot(record_model, connection, table_name)
|
|
128
|
+
count = -> { record_model.count }
|
|
129
|
+
event_count = solid_cache? ? RecordCount.new(connection, table_name).solid_cache_count(&count) : count.call
|
|
75
130
|
|
|
76
131
|
available_snapshot(
|
|
77
|
-
db_size_bytes:
|
|
78
|
-
event_count:
|
|
132
|
+
db_size_bytes: DatabaseSize.call(connection: connection, table_name: table_name),
|
|
133
|
+
event_count: event_count,
|
|
134
|
+
**extras_for(record_model)
|
|
79
135
|
)
|
|
80
136
|
end
|
|
81
137
|
|
|
82
|
-
def
|
|
83
|
-
|
|
138
|
+
def extras_for(record_model)
|
|
139
|
+
solid_cable? ? solid_cable_extras(record_model) : {}
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def data_source_available?(connection, table_name)
|
|
143
|
+
!table_name.empty? && connection.data_source_exists?(table_name)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def solid_cable_extras(record_model)
|
|
147
|
+
{
|
|
148
|
+
trimmable_count: safe_query { record_model.trimmable.count },
|
|
149
|
+
oldest_message_age_seconds: safe_query { (Time.current - record_model.minimum(:created_at).to_time).to_i }
|
|
150
|
+
}
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def safe_query
|
|
154
|
+
yield
|
|
155
|
+
rescue *StorageInfoSnapshot::CONNECTION_ERRORS, TypeError, NoMethodError
|
|
156
|
+
nil
|
|
84
157
|
end
|
|
85
158
|
end
|
|
86
159
|
|
|
@@ -105,6 +178,20 @@ module SolidObserver
|
|
|
105
178
|
record_label: "cache rows",
|
|
106
179
|
model: -> { ::SolidCache::Entry },
|
|
107
180
|
enabled: -> { SolidObserver.config.solid_cache_enabled? }
|
|
181
|
+
),
|
|
182
|
+
Component.new(
|
|
183
|
+
key: "cable_observer",
|
|
184
|
+
label: "Cable telemetry",
|
|
185
|
+
record_label: "observer events",
|
|
186
|
+
model: -> { SolidObserver::CableEvent },
|
|
187
|
+
enabled: -> { SolidObserver.config.solid_cable_enabled? }
|
|
188
|
+
),
|
|
189
|
+
Component.new(
|
|
190
|
+
key: "solid_cable",
|
|
191
|
+
label: "Solid Cable messages",
|
|
192
|
+
record_label: "messages",
|
|
193
|
+
model: -> { ::SolidCable::Message },
|
|
194
|
+
enabled: -> { SolidObserver.config.solid_cable_enabled? }
|
|
108
195
|
)
|
|
109
196
|
].freeze
|
|
110
197
|
|
|
@@ -113,6 +200,7 @@ module SolidObserver
|
|
|
113
200
|
ActiveRecord::StatementInvalid,
|
|
114
201
|
*([PG::ConnectionBad] if defined?(PG::ConnectionBad)),
|
|
115
202
|
*([Mysql2::Error::ConnectionError] if defined?(Mysql2::Error::ConnectionError)),
|
|
203
|
+
*([Trilogy::Error] if defined?(Trilogy::Error)),
|
|
116
204
|
*([SQLite3::CantOpenException] if defined?(SQLite3::CantOpenException))
|
|
117
205
|
].freeze
|
|
118
206
|
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidObserver
|
|
4
|
+
module Services
|
|
5
|
+
class UnifiedFeed
|
|
6
|
+
SAFE_KEYS = %i[component event_type recorded_at detail correlation_id error_class].freeze
|
|
7
|
+
|
|
8
|
+
def self.call(limit: 20)
|
|
9
|
+
new(limit: limit).call
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def initialize(limit:)
|
|
13
|
+
@limit = limit
|
|
14
|
+
@config = SolidObserver.config
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def call
|
|
18
|
+
return [] if @config.realtime_mode?
|
|
19
|
+
|
|
20
|
+
rows = []
|
|
21
|
+
rows.concat(queue_rows) if @config.solid_queue_enabled?
|
|
22
|
+
rows.concat(cache_rows) if @config.solid_cache_enabled?
|
|
23
|
+
rows.concat(cable_rows) if @config.solid_cable_enabled?
|
|
24
|
+
|
|
25
|
+
rows.sort_by { |row| row[:recorded_at] || Time.at(0) }.reverse.first(@limit)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def queue_rows
|
|
31
|
+
QueueEvent.recent(@limit).map { |e| normalize_queue(e) }
|
|
32
|
+
rescue => error
|
|
33
|
+
log_error(:queue, error)
|
|
34
|
+
[]
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def cache_rows
|
|
38
|
+
CacheEvent.recent(@limit).map { |e| normalize_cache(e) }
|
|
39
|
+
rescue => error
|
|
40
|
+
log_error(:cache, error)
|
|
41
|
+
[]
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def cable_rows
|
|
45
|
+
CableEvent.recent(@limit).map { |e| normalize_cable(e) }
|
|
46
|
+
rescue => error
|
|
47
|
+
log_error(:cable, error)
|
|
48
|
+
[]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def normalize_queue(event)
|
|
52
|
+
{
|
|
53
|
+
component: :queue,
|
|
54
|
+
event_type: event.event_type,
|
|
55
|
+
recorded_at: event.recorded_at,
|
|
56
|
+
detail: [event.job_class, event.queue_name].compact.join(" · "),
|
|
57
|
+
correlation_id: event.correlation_id,
|
|
58
|
+
error_class: nil # queue events have no error_class column
|
|
59
|
+
}
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def normalize_cache(event)
|
|
63
|
+
{
|
|
64
|
+
component: :cache,
|
|
65
|
+
event_type: event.event_type,
|
|
66
|
+
recorded_at: event.recorded_at,
|
|
67
|
+
detail: cache_detail(event),
|
|
68
|
+
correlation_id: event.correlation_id,
|
|
69
|
+
error_class: event.error_class
|
|
70
|
+
}
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def normalize_cable(event)
|
|
74
|
+
{
|
|
75
|
+
component: :cable,
|
|
76
|
+
event_type: event.event_type,
|
|
77
|
+
recorded_at: event.recorded_at,
|
|
78
|
+
detail: cable_detail(event),
|
|
79
|
+
correlation_id: event.correlation_id,
|
|
80
|
+
error_class: event.error_class
|
|
81
|
+
}
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def cache_detail(event)
|
|
85
|
+
digest = event.key_digest.to_s
|
|
86
|
+
digest = "#{digest.first(10)}…" if digest.length > 10
|
|
87
|
+
digest
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def cable_detail(event)
|
|
91
|
+
event.channel_class.presence || event.broadcasting_digest.to_s.first(10)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def log_error(component, error)
|
|
95
|
+
return unless defined?(Rails) && Rails.logger
|
|
96
|
+
|
|
97
|
+
Rails.logger.warn("UnifiedFeed #{component} error: #{error.message}")
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
data/lib/solid_observer.rb
CHANGED
|
@@ -1,30 +1,61 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "active_support/lazy_load_hooks"
|
|
4
|
+
|
|
3
5
|
require_relative "solid_observer/version"
|
|
4
6
|
require_relative "solid_observer/configuration"
|
|
5
7
|
require_relative "solid_observer/correlation_id_resolver"
|
|
6
|
-
require_relative "solid_observer/
|
|
7
|
-
require_relative "solid_observer/base_metric" if defined?(ActiveRecord)
|
|
8
|
+
require_relative "solid_observer/correlated"
|
|
8
9
|
require_relative "solid_observer/params/jobs_filter"
|
|
9
10
|
require_relative "solid_observer/params/events_filter"
|
|
10
|
-
require_relative "solid_observer/queries/job_executions_query" if defined?(ActiveRecord)
|
|
11
|
-
require_relative "solid_observer/queries/events_query" if defined?(ActiveRecord)
|
|
12
|
-
require_relative "solid_observer/queries/execution_finder" if defined?(ActiveRecord)
|
|
13
|
-
require_relative "solid_observer/services/record_event" if defined?(ActiveRecord)
|
|
14
|
-
require_relative "solid_observer/services/flush_event_buffer" if defined?(ActiveRecord)
|
|
15
|
-
require_relative "solid_observer/services/cleanup_storage" if defined?(ActiveRecord)
|
|
16
11
|
require_relative "solid_observer/services/ui_auth_check"
|
|
17
|
-
require_relative "solid_observer/
|
|
18
|
-
require_relative "solid_observer/subscriber" if defined?(ActiveSupport)
|
|
12
|
+
require_relative "solid_observer/subscriber"
|
|
19
13
|
require_relative "solid_observer/cli/base"
|
|
20
14
|
require_relative "solid_observer/cli/status"
|
|
21
15
|
require_relative "solid_observer/cli/storage"
|
|
22
16
|
require_relative "solid_observer/cli/jobs"
|
|
17
|
+
require_relative "solid_observer/cli/trace"
|
|
18
|
+
require_relative "solid_observer/cli/health"
|
|
23
19
|
require_relative "solid_observer/queue_stats"
|
|
24
|
-
require_relative "../app/presenters/solid_observer/execution_presenter"
|
|
25
20
|
require_relative "solid_observer/engine" if defined?(Rails::Engine)
|
|
26
21
|
|
|
22
|
+
ActiveSupport.on_load(:active_record) do
|
|
23
|
+
require_relative "solid_observer/base_record"
|
|
24
|
+
require_relative "solid_observer/base_event"
|
|
25
|
+
require_relative "solid_observer/base_metric"
|
|
26
|
+
require_relative "solid_observer/queries/job_executions_query"
|
|
27
|
+
require_relative "solid_observer/queries/events_query"
|
|
28
|
+
require_relative "solid_observer/queries/trace_query"
|
|
29
|
+
require_relative "solid_observer/queries/execution_finder"
|
|
30
|
+
require_relative "solid_observer/services/record_event"
|
|
31
|
+
require_relative "solid_observer/services/flush_event_buffer"
|
|
32
|
+
require_relative "solid_observer/services/cleanup_storage"
|
|
33
|
+
require_relative "solid_observer/queue_event_buffer"
|
|
34
|
+
require_relative "solid_observer/cache_event_buffer"
|
|
35
|
+
require_relative "solid_observer/services/flush_cache_metrics"
|
|
36
|
+
require_relative "solid_observer/cache_metric_buffer"
|
|
37
|
+
require_relative "solid_observer/cache_subscriber"
|
|
38
|
+
require_relative "solid_observer/services/record_cache_event"
|
|
39
|
+
require_relative "solid_observer/services/record_cache_metric"
|
|
40
|
+
require_relative "solid_observer/services/flush_cache_event_buffer"
|
|
41
|
+
require_relative "solid_observer/services/cache_stats"
|
|
42
|
+
require_relative "solid_observer/services/cache_operations"
|
|
43
|
+
require_relative "solid_observer/cable_event_buffer"
|
|
44
|
+
require_relative "solid_observer/services/flush_cable_metrics"
|
|
45
|
+
require_relative "solid_observer/cable_metric_buffer"
|
|
46
|
+
require_relative "solid_observer/cable_subscriber"
|
|
47
|
+
require_relative "solid_observer/services/record_cable_event"
|
|
48
|
+
require_relative "solid_observer/services/record_cable_metric"
|
|
49
|
+
require_relative "solid_observer/services/flush_cable_event_buffer"
|
|
50
|
+
require_relative "solid_observer/services/cable_stats"
|
|
51
|
+
require_relative "solid_observer/services/cable_operations"
|
|
52
|
+
require_relative "solid_observer/services/health_score"
|
|
53
|
+
require_relative "solid_observer/services/unified_feed"
|
|
54
|
+
end
|
|
55
|
+
|
|
27
56
|
module SolidObserver
|
|
57
|
+
autoload :ExecutionPresenter, File.expand_path("../app/presenters/solid_observer/execution_presenter", __dir__)
|
|
58
|
+
|
|
28
59
|
class Error < StandardError; end
|
|
29
60
|
|
|
30
61
|
class << self
|
|
@@ -40,30 +40,48 @@ namespace :solid_observer do
|
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
namespace :buffer do
|
|
43
|
-
desc "Flush the event
|
|
43
|
+
desc "Flush the event buffers to the database"
|
|
44
44
|
task flush: :environment do
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
buffers = [
|
|
46
|
+
{label: "queue events", buffer: SolidObserver::QueueEventBuffer.instance},
|
|
47
|
+
{label: "cache events", buffer: SolidObserver::CacheEventBuffer.instance},
|
|
48
|
+
{label: "cache metric buckets", buffer: SolidObserver::CacheMetricBuffer.instance}
|
|
49
|
+
]
|
|
50
|
+
buffer_sizes = buffers.to_h { |entry| [entry, entry[:buffer].size] }
|
|
47
51
|
|
|
48
|
-
if
|
|
49
|
-
puts "
|
|
52
|
+
if buffer_sizes.values.sum.zero?
|
|
53
|
+
puts "Buffers are empty, nothing to flush"
|
|
50
54
|
else
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
55
|
+
buffers.each do |entry|
|
|
56
|
+
size = buffer_sizes.fetch(entry)
|
|
57
|
+
next if size.zero?
|
|
58
|
+
|
|
59
|
+
puts "Flushing #{size} #{entry[:label]} from buffer..."
|
|
60
|
+
entry[:buffer].flush!
|
|
61
|
+
end
|
|
62
|
+
puts "✓ Buffers flushed successfully"
|
|
54
63
|
end
|
|
55
64
|
end
|
|
56
65
|
|
|
57
|
-
desc "Clear the event
|
|
66
|
+
desc "Clear the event buffers without flushing to database"
|
|
58
67
|
task clear: :environment do
|
|
59
|
-
|
|
60
|
-
|
|
68
|
+
buffers = [
|
|
69
|
+
{label: "queue events", buffer: SolidObserver::QueueEventBuffer.instance},
|
|
70
|
+
{label: "cache events", buffer: SolidObserver::CacheEventBuffer.instance},
|
|
71
|
+
{label: "cache metric buckets", buffer: SolidObserver::CacheMetricBuffer.instance}
|
|
72
|
+
]
|
|
73
|
+
buffer_sizes = buffers.to_h { |entry| [entry, entry[:buffer].size] }
|
|
61
74
|
|
|
62
|
-
if
|
|
63
|
-
puts "
|
|
75
|
+
if buffer_sizes.values.sum.zero?
|
|
76
|
+
puts "Buffers are already empty"
|
|
64
77
|
else
|
|
65
|
-
|
|
66
|
-
|
|
78
|
+
buffers.each do |entry|
|
|
79
|
+
size = buffer_sizes.fetch(entry)
|
|
80
|
+
next if size.zero?
|
|
81
|
+
|
|
82
|
+
entry[:buffer].clear
|
|
83
|
+
puts "✓ Cleared #{size} #{entry[:label]} from buffer (not saved to database)"
|
|
84
|
+
end
|
|
67
85
|
end
|
|
68
86
|
end
|
|
69
87
|
end
|
|
@@ -97,6 +115,18 @@ namespace :solid_observer do
|
|
|
97
115
|
end
|
|
98
116
|
end
|
|
99
117
|
|
|
118
|
+
namespace :cable do
|
|
119
|
+
desc "Trim expired Solid Cable messages"
|
|
120
|
+
task trim: :environment do
|
|
121
|
+
if !SolidObserver::Services::CableOperations.available?
|
|
122
|
+
puts SolidObserver::Services::CableOperations.unavailable_message
|
|
123
|
+
next
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
puts SolidObserver::Services::CableOperations.trim[:message]
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
100
130
|
namespace :storage do
|
|
101
131
|
desc "Run storage cleanup based on retention policy"
|
|
102
132
|
task cleanup: :environment do
|
|
@@ -104,7 +134,7 @@ namespace :solid_observer do
|
|
|
104
134
|
puts "Running storage cleanup (retention: #{retention.inspect})..."
|
|
105
135
|
|
|
106
136
|
deleted_count = SolidObserver::Services::CleanupStorage.call
|
|
107
|
-
puts "✓ Cleanup complete: #{deleted_count} old
|
|
137
|
+
puts "✓ Cleanup complete: #{deleted_count} old telemetry rows deleted"
|
|
108
138
|
end
|
|
109
139
|
|
|
110
140
|
desc "Purge ALL SolidObserver storage data (use with caution!)"
|
|
@@ -114,10 +144,30 @@ namespace :solid_observer do
|
|
|
114
144
|
|
|
115
145
|
confirmation = $stdin.gets&.strip&.downcase
|
|
116
146
|
if confirmation == "y"
|
|
117
|
-
|
|
147
|
+
data_source_exists = lambda do |model|
|
|
148
|
+
table_name = model.table_name.to_s
|
|
149
|
+
!table_name.empty? && model.connection.data_source_exists?(table_name)
|
|
150
|
+
rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::StatementInvalid, TypeError
|
|
151
|
+
false
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
telemetry_models = [
|
|
155
|
+
{label: :queue_events, model: SolidObserver::QueueEvent},
|
|
156
|
+
{label: :cache_events, model: SolidObserver::CacheEvent},
|
|
157
|
+
{label: :cache_metrics, model: SolidObserver::CacheMetric}
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
telemetry_counts = telemetry_models.to_h do |entry|
|
|
161
|
+
count = data_source_exists.call(entry[:model]) ? entry[:model].count : 0
|
|
162
|
+
[entry[:label], count]
|
|
163
|
+
end
|
|
118
164
|
snapshot_count = SolidObserver::StorageInfo.count
|
|
119
165
|
|
|
120
|
-
|
|
166
|
+
telemetry_models.each do |entry|
|
|
167
|
+
next unless data_source_exists.call(entry[:model])
|
|
168
|
+
|
|
169
|
+
entry[:model].delete_all
|
|
170
|
+
end
|
|
121
171
|
SolidObserver::StorageInfo.delete_all
|
|
122
172
|
|
|
123
173
|
connection = SolidObserver::QueueEvent.connection
|
|
@@ -126,12 +176,23 @@ namespace :solid_observer do
|
|
|
126
176
|
connection.execute("VACUUM")
|
|
127
177
|
puts "✓ Database vacuumed to reclaim disk space"
|
|
128
178
|
when "postgresql"
|
|
129
|
-
|
|
130
|
-
|
|
179
|
+
[
|
|
180
|
+
SolidObserver::QueueEvent.table_name,
|
|
181
|
+
SolidObserver::StorageInfo.table_name,
|
|
182
|
+
(SolidObserver::CacheEvent.table_name if data_source_exists.call(SolidObserver::CacheEvent)),
|
|
183
|
+
(SolidObserver::CacheMetric.table_name if data_source_exists.call(SolidObserver::CacheMetric))
|
|
184
|
+
].compact.each do |table_name|
|
|
185
|
+
connection.execute("ANALYZE #{table_name}")
|
|
186
|
+
end
|
|
131
187
|
puts "✓ Database tables analyzed"
|
|
132
188
|
end
|
|
133
189
|
|
|
134
|
-
puts
|
|
190
|
+
puts(
|
|
191
|
+
"✓ Purged #{telemetry_counts.fetch(:queue_events)} queue events, " \
|
|
192
|
+
"#{telemetry_counts.fetch(:cache_events)} cache events, " \
|
|
193
|
+
"#{telemetry_counts.fetch(:cache_metrics)} cache metrics, " \
|
|
194
|
+
"and #{snapshot_count} storage snapshots"
|
|
195
|
+
)
|
|
135
196
|
else
|
|
136
197
|
puts "Aborted"
|
|
137
198
|
end
|
|
@@ -143,11 +204,26 @@ namespace :solid_observer do
|
|
|
143
204
|
SolidObserver::CLI::Status.call
|
|
144
205
|
end
|
|
145
206
|
|
|
207
|
+
desc "Display aggregate and per-component health status"
|
|
208
|
+
task health: :environment do
|
|
209
|
+
SolidObserver::CLI::Health.call
|
|
210
|
+
end
|
|
211
|
+
|
|
146
212
|
desc "Display storage information and database statistics"
|
|
147
213
|
task storage: :environment do
|
|
148
214
|
SolidObserver::CLI::Storage.call
|
|
149
215
|
end
|
|
150
216
|
|
|
217
|
+
desc "Show the cross-component trace for a correlation_id"
|
|
218
|
+
task :trace, [:correlation_id, :limit] => :environment do |_t, args|
|
|
219
|
+
if args[:correlation_id].nil?
|
|
220
|
+
puts "Error: correlation_id required. Usage: rails solid_observer:trace[CORRELATION_ID,LIMIT]"
|
|
221
|
+
exit 1
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
SolidObserver::CLI::Trace.call(correlation_id: args[:correlation_id], limit: args[:limit])
|
|
225
|
+
end
|
|
226
|
+
|
|
151
227
|
namespace :jobs do
|
|
152
228
|
desc "List jobs with optional filters (status, queue, class, limit)"
|
|
153
229
|
task :list, [:status, :queue, :job_class, :limit] => :environment do |_t, args|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: solid_observer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- BartOz
|
|
@@ -50,20 +50,24 @@ files:
|
|
|
50
50
|
- LICENSE.txt
|
|
51
51
|
- README.md
|
|
52
52
|
- app/assets/javascripts/solid_observer/live_poll.js
|
|
53
|
-
- app/assets/stylesheets/solid_observer/application.css
|
|
54
53
|
- app/controllers/concerns/solid_observer/paginatable.rb
|
|
55
54
|
- app/controllers/concerns/solid_observer/require_persistence_mode.rb
|
|
56
55
|
- app/controllers/concerns/solid_observer/require_solid_queue.rb
|
|
57
56
|
- app/controllers/solid_observer/application_controller.rb
|
|
57
|
+
- app/controllers/solid_observer/cable_dashboard_controller.rb
|
|
58
|
+
- app/controllers/solid_observer/cable_operations_controller.rb
|
|
58
59
|
- app/controllers/solid_observer/cache_dashboard_controller.rb
|
|
59
60
|
- app/controllers/solid_observer/cache_operations_controller.rb
|
|
60
61
|
- app/controllers/solid_observer/dashboard_controller.rb
|
|
61
62
|
- app/controllers/solid_observer/events_controller.rb
|
|
62
63
|
- app/controllers/solid_observer/jobs_controller.rb
|
|
63
64
|
- app/controllers/solid_observer/storages_controller.rb
|
|
65
|
+
- app/controllers/solid_observer/traces_controller.rb
|
|
64
66
|
- app/helpers/solid_observer/application_helper.rb
|
|
65
67
|
- app/helpers/solid_observer/dashboard_helper.rb
|
|
66
68
|
- app/jobs/solid_observer/cleanup_job.rb
|
|
69
|
+
- app/models/solid_observer/cable_event.rb
|
|
70
|
+
- app/models/solid_observer/cable_metric.rb
|
|
67
71
|
- app/models/solid_observer/cache_event.rb
|
|
68
72
|
- app/models/solid_observer/cache_metric.rb
|
|
69
73
|
- app/models/solid_observer/queue_event.rb
|
|
@@ -71,6 +75,10 @@ files:
|
|
|
71
75
|
- app/models/solid_observer/storage_info.rb
|
|
72
76
|
- app/presenters/solid_observer/execution_presenter.rb
|
|
73
77
|
- app/views/layouts/solid_observer/application.html.erb
|
|
78
|
+
- app/views/solid_observer/cable_dashboard/_charts.html.erb
|
|
79
|
+
- app/views/solid_observer/cable_dashboard/_recent_events.html.erb
|
|
80
|
+
- app/views/solid_observer/cable_dashboard/_summary.html.erb
|
|
81
|
+
- app/views/solid_observer/cable_dashboard/index.html.erb
|
|
74
82
|
- app/views/solid_observer/cache_dashboard/_charts.html.erb
|
|
75
83
|
- app/views/solid_observer/cache_dashboard/_recent_events.html.erb
|
|
76
84
|
- app/views/solid_observer/cache_dashboard/_summary.html.erb
|
|
@@ -78,10 +86,15 @@ files:
|
|
|
78
86
|
- app/views/solid_observer/cache_operations/_confirm_clear.html.erb
|
|
79
87
|
- app/views/solid_observer/cache_operations/index.html.erb
|
|
80
88
|
- app/views/solid_observer/dashboard/_chart.html.erb
|
|
89
|
+
- app/views/solid_observer/dashboard/_component_cards.html.erb
|
|
90
|
+
- app/views/solid_observer/dashboard/_health.html.erb
|
|
91
|
+
- app/views/solid_observer/dashboard/_home.html.erb
|
|
81
92
|
- app/views/solid_observer/dashboard/_live_state.html.erb
|
|
93
|
+
- app/views/solid_observer/dashboard/_queue.html.erb
|
|
82
94
|
- app/views/solid_observer/dashboard/_queue_table.html.erb
|
|
83
95
|
- app/views/solid_observer/dashboard/_right_now.html.erb
|
|
84
96
|
- app/views/solid_observer/dashboard/_throughput.html.erb
|
|
97
|
+
- app/views/solid_observer/dashboard/_unified_feed.html.erb
|
|
85
98
|
- app/views/solid_observer/dashboard/index.html.erb
|
|
86
99
|
- app/views/solid_observer/errors/storage_unavailable.html.erb
|
|
87
100
|
- app/views/solid_observer/events/index.html.erb
|
|
@@ -92,9 +105,7 @@ files:
|
|
|
92
105
|
- app/views/solid_observer/shared/_pagination.html.erb
|
|
93
106
|
- app/views/solid_observer/shared/_stat_card.html.erb
|
|
94
107
|
- app/views/solid_observer/storages/show.html.erb
|
|
95
|
-
-
|
|
96
|
-
- bin/quality_gate
|
|
97
|
-
- bin/setup
|
|
108
|
+
- app/views/solid_observer/traces/show.html.erb
|
|
98
109
|
- config/routes.rb
|
|
99
110
|
- db/migrate/20260115000001_create_solid_observer_queue_events.rb
|
|
100
111
|
- db/migrate/20260115000002_create_solid_observer_metrics.rb
|
|
@@ -103,40 +114,64 @@ files:
|
|
|
103
114
|
- db/migrate/20260601000001_create_solid_observer_cache_events.rb
|
|
104
115
|
- db/migrate/20260601000002_create_solid_observer_cache_metrics.rb
|
|
105
116
|
- db/migrate/20260602000001_add_component_to_solid_observer_storage_infos.rb
|
|
117
|
+
- db/migrate/20260612000001_add_event_type_recorded_at_index_to_cache_events.rb
|
|
118
|
+
- db/migrate/20260619000001_create_solid_observer_cable_events.rb
|
|
119
|
+
- db/migrate/20260619000002_create_solid_observer_cable_metrics.rb
|
|
120
|
+
- db/migrate/20260630000001_add_correlation_id_to_cache_events.rb
|
|
121
|
+
- db/migrate/20260630000002_add_correlation_id_to_cable_events.rb
|
|
106
122
|
- lib/generators/solid_observer/install_generator.rb
|
|
107
123
|
- lib/generators/solid_observer/templates/initializer.rb.tt
|
|
108
124
|
- lib/solid_observer.rb
|
|
109
125
|
- lib/solid_observer/base_event.rb
|
|
110
126
|
- lib/solid_observer/base_metric.rb
|
|
127
|
+
- lib/solid_observer/base_record.rb
|
|
128
|
+
- lib/solid_observer/cable_event_buffer.rb
|
|
129
|
+
- lib/solid_observer/cable_metric_buffer.rb
|
|
130
|
+
- lib/solid_observer/cable_subscriber.rb
|
|
111
131
|
- lib/solid_observer/cache_event_buffer.rb
|
|
132
|
+
- lib/solid_observer/cache_metric_buffer.rb
|
|
112
133
|
- lib/solid_observer/cache_subscriber.rb
|
|
113
134
|
- lib/solid_observer/chart_buffer.rb
|
|
114
135
|
- lib/solid_observer/cli/base.rb
|
|
136
|
+
- lib/solid_observer/cli/health.rb
|
|
115
137
|
- lib/solid_observer/cli/jobs.rb
|
|
116
138
|
- lib/solid_observer/cli/status.rb
|
|
117
139
|
- lib/solid_observer/cli/storage.rb
|
|
140
|
+
- lib/solid_observer/cli/trace.rb
|
|
118
141
|
- lib/solid_observer/configuration.rb
|
|
142
|
+
- lib/solid_observer/correlated.rb
|
|
119
143
|
- lib/solid_observer/correlation_id_resolver.rb
|
|
120
144
|
- lib/solid_observer/engine.rb
|
|
145
|
+
- lib/solid_observer/event_buffer_core.rb
|
|
121
146
|
- lib/solid_observer/params/events_filter.rb
|
|
122
147
|
- lib/solid_observer/params/jobs_filter.rb
|
|
123
148
|
- lib/solid_observer/queries/events_query.rb
|
|
124
149
|
- lib/solid_observer/queries/execution_finder.rb
|
|
125
150
|
- lib/solid_observer/queries/job_executions_query.rb
|
|
151
|
+
- lib/solid_observer/queries/trace_query.rb
|
|
126
152
|
- lib/solid_observer/queue_event_buffer.rb
|
|
127
153
|
- lib/solid_observer/queue_stats.rb
|
|
154
|
+
- lib/solid_observer/services/cable_operations.rb
|
|
155
|
+
- lib/solid_observer/services/cable_stats.rb
|
|
128
156
|
- lib/solid_observer/services/cache_operations.rb
|
|
129
157
|
- lib/solid_observer/services/cache_stats.rb
|
|
130
158
|
- lib/solid_observer/services/cleanup_storage.rb
|
|
131
159
|
- lib/solid_observer/services/database_size.rb
|
|
160
|
+
- lib/solid_observer/services/flush_cable_event_buffer.rb
|
|
161
|
+
- lib/solid_observer/services/flush_cable_metrics.rb
|
|
132
162
|
- lib/solid_observer/services/flush_cache_event_buffer.rb
|
|
163
|
+
- lib/solid_observer/services/flush_cache_metrics.rb
|
|
133
164
|
- lib/solid_observer/services/flush_event_buffer.rb
|
|
165
|
+
- lib/solid_observer/services/health_score.rb
|
|
134
166
|
- lib/solid_observer/services/install_migrations.rb
|
|
167
|
+
- lib/solid_observer/services/record_cable_event.rb
|
|
168
|
+
- lib/solid_observer/services/record_cable_metric.rb
|
|
135
169
|
- lib/solid_observer/services/record_cache_event.rb
|
|
136
170
|
- lib/solid_observer/services/record_cache_metric.rb
|
|
137
171
|
- lib/solid_observer/services/record_event.rb
|
|
138
172
|
- lib/solid_observer/services/storage_info_snapshot.rb
|
|
139
173
|
- lib/solid_observer/services/ui_auth_check.rb
|
|
174
|
+
- lib/solid_observer/services/unified_feed.rb
|
|
140
175
|
- lib/solid_observer/subscriber.rb
|
|
141
176
|
- lib/solid_observer/version.rb
|
|
142
177
|
- lib/tasks/solid_observer.rake
|
|
@@ -163,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
163
198
|
- !ruby/object:Gem::Version
|
|
164
199
|
version: '0'
|
|
165
200
|
requirements: []
|
|
166
|
-
rubygems_version:
|
|
201
|
+
rubygems_version: 3.6.9
|
|
167
202
|
specification_version: 4
|
|
168
203
|
summary: Observability for the Rails 8's Solid Stack
|
|
169
204
|
test_files: []
|