solid_observer 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +25 -10
- data/app/controllers/solid_observer/dashboard_controller.rb +20 -6
- data/app/controllers/solid_observer/traces_controller.rb +17 -0
- data/app/helpers/solid_observer/application_helper.rb +31 -3
- data/app/views/layouts/solid_observer/application.html.erb +16 -1
- data/app/views/solid_observer/cable_dashboard/_recent_events.html.erb +6 -1
- data/app/views/solid_observer/cache_dashboard/_recent_events.html.erb +6 -1
- data/app/views/solid_observer/dashboard/_component_cards.html.erb +18 -0
- data/app/views/solid_observer/dashboard/_health.html.erb +6 -0
- data/app/views/solid_observer/dashboard/_home.html.erb +10 -0
- data/app/views/solid_observer/dashboard/_queue.html.erb +138 -0
- data/app/views/solid_observer/dashboard/_unified_feed.html.erb +38 -0
- data/app/views/solid_observer/dashboard/index.html.erb +5 -138
- data/app/views/solid_observer/events/index.html.erb +6 -1
- data/app/views/solid_observer/traces/show.html.erb +80 -0
- data/config/routes.rb +1 -0
- data/db/migrate/20260630000001_add_correlation_id_to_cache_events.rb +8 -0
- data/db/migrate/20260630000002_add_correlation_id_to_cable_events.rb +8 -0
- data/lib/generators/solid_observer/templates/initializer.rb.tt +6 -2
- data/lib/solid_observer/cli/health.rb +49 -0
- data/lib/solid_observer/cli/trace.rb +106 -0
- data/lib/solid_observer/configuration.rb +1 -4
- data/lib/solid_observer/correlated.rb +23 -0
- data/lib/solid_observer/queries/trace_query.rb +151 -0
- data/lib/solid_observer/services/health_score.rb +85 -0
- data/lib/solid_observer/services/record_cable_event.rb +1 -0
- data/lib/solid_observer/services/record_cache_event.rb +1 -0
- data/lib/solid_observer/services/unified_feed.rb +101 -0
- data/lib/solid_observer/version.rb +1 -1
- data/lib/solid_observer.rb +6 -0
- data/lib/tasks/solid_observer.rake +15 -0
- metadata +17 -2
|
@@ -1,140 +1,7 @@
|
|
|
1
1
|
<% content_for :title, "Dashboard" %>
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
<h1>Queue overview</h1>
|
|
9
|
-
<div class="so-queue-overview__intro">
|
|
10
|
-
<span class="so-badge so-badge--pill <%= queue_status_class %>">
|
|
11
|
-
<svg class="so-badge__dot" viewBox="0 0 6 6" aria-hidden="true">
|
|
12
|
-
<circle r="3" cx="3" cy="3" />
|
|
13
|
-
</svg>
|
|
14
|
-
<%= queue_enabled ? "Available" : "Unavailable" %>
|
|
15
|
-
</span>
|
|
16
|
-
<% if queue_enabled %>
|
|
17
|
-
<p class="so-queue-overview__hint">Selected range: <%= range_label(@range) %></p>
|
|
18
|
-
<% end %>
|
|
19
|
-
</div>
|
|
20
|
-
</div>
|
|
21
|
-
|
|
22
|
-
<% if !queue_enabled %>
|
|
23
|
-
<section class="so-card so-card--section" aria-labelledby="so-queue-unavailable-heading">
|
|
24
|
-
<h2 id="so-queue-unavailable-heading">Queue Unavailable</h2>
|
|
25
|
-
<p class="so-dashboard-section__meta so-dashboard-section__meta--idle">Queue component is disabled or not installed. Enable SolidQueue to access dashboard controls.</p>
|
|
26
|
-
</section>
|
|
27
|
-
<% else %>
|
|
28
|
-
<%# Zone A: Toolbar — range selector, Refresh, Live toggle, help disclosure %>
|
|
29
|
-
<div class="so-dashboard-toolbar" data-so-live data-so-poll-url="<%= poll_data_path %>">
|
|
30
|
-
<form action="<%= request.path %>" method="get" class="so-dashboard-toolbar__left">
|
|
31
|
-
<label for="so-range" class="so-card__label so-filters__label">Range</label>
|
|
32
|
-
<select id="so-range" name="range" data-so-range-select onchange="this.form.submit()">
|
|
33
|
-
<% SolidObserver::QueueStats::RANGES.each_key do |key| %>
|
|
34
|
-
<option value="<%= key %>" <%= "selected" if key == @range %>><%= key %></option>
|
|
35
|
-
<% end %>
|
|
36
|
-
</select>
|
|
37
|
-
<button type="button" class="so-btn so-btn--refresh" data-so-refresh aria-busy="false">Refresh data</button>
|
|
38
|
-
<span class="so-toolbar-freshness" data-so-freshness aria-live="polite"></span>
|
|
39
|
-
<input type="hidden" name="live" value="<%= @live ? 'on' : '' %>">
|
|
40
|
-
</form>
|
|
41
|
-
<div class="so-dashboard-toolbar__right">
|
|
42
|
-
<label class="so-toggle so-toggle--pill <%= "so-toggle--on" if @live %>">
|
|
43
|
-
<input type="checkbox" name="live" value="on" <%= "checked" if @live %> data-so-live-toggle>
|
|
44
|
-
<span class="so-toggle__track" aria-hidden="true">
|
|
45
|
-
<span class="so-toggle__thumb"></span>
|
|
46
|
-
</span>
|
|
47
|
-
<span class="so-toggle__label">Live</span>
|
|
48
|
-
<span class="so-toggle__sep" aria-hidden="true"> · </span>
|
|
49
|
-
<span class="so-toggle__cadence" aria-live="polite"><%= @live ? "5s" : "off" %></span>
|
|
50
|
-
<span class="so-toggle__dot" aria-hidden="true"></span>
|
|
51
|
-
</label>
|
|
52
|
-
<div class="so-help-wrapper" data-so-help-wrapper>
|
|
53
|
-
<button type="button" class="so-btn so-btn--help" data-so-help-btn aria-expanded="false" aria-controls="so-help-panel">(?)
|
|
54
|
-
</button>
|
|
55
|
-
<div id="so-help-panel" class="so-help-panel" data-so-help-panel hidden>
|
|
56
|
-
<p>Live mode polls every 5 seconds for current queue state. Throughput and chart data update only on range change or manual refresh.</p>
|
|
57
|
-
</div>
|
|
58
|
-
</div>
|
|
59
|
-
</div>
|
|
60
|
-
</div>
|
|
61
|
-
|
|
62
|
-
<%# Zone B: Right now — live state %>
|
|
63
|
-
<section class="so-dashboard-section" aria-labelledby="so-live-state-heading">
|
|
64
|
-
<div class="so-dashboard-section__header">
|
|
65
|
-
<h2 class="so-dashboard-section__title" id="so-live-state-heading">Right now</h2>
|
|
66
|
-
<span class="so-dashboard-section__meta"><span class="so-badge so-badge--pill so-badge--success"><svg class="so-badge__dot" viewBox="0 0 6 6" aria-hidden="true"><circle r="3" cx="3" cy="3" fill="var(--so-live-marker, var(--so-success))"/></svg></span> current</span>
|
|
67
|
-
</div>
|
|
68
|
-
<%= render "live_state", stats: @stats %>
|
|
69
|
-
</section>
|
|
70
|
-
|
|
71
|
-
<%# Zone C: Throughput in selected range %>
|
|
72
|
-
<% if persistence_mode? && @stats[:available] %>
|
|
73
|
-
<section class="so-dashboard-section" aria-labelledby="so-throughput-heading">
|
|
74
|
-
<div class="so-dashboard-section__header">
|
|
75
|
-
<h2 class="so-dashboard-section__title" id="so-throughput-heading">Throughput in selected range</h2>
|
|
76
|
-
<span class="so-dashboard-section__meta"><span class="so-badge so-badge--pill so-badge--info"><svg class="so-badge__dot" viewBox="0 0 6 6" aria-hidden="true"><circle r="3" cx="3" cy="3" fill="var(--so-range-marker, var(--so-info))"/></svg></span> <span data-so-range-copy><%= range_label(@range) %></span></span>
|
|
77
|
-
</div>
|
|
78
|
-
<%= render "throughput", stats: @stats, range: @range %>
|
|
79
|
-
</section>
|
|
80
|
-
<% end %>
|
|
81
|
-
|
|
82
|
-
<%# Zone D: Charts %>
|
|
83
|
-
<% if @stats[:available] %>
|
|
84
|
-
<section class="so-dashboard-section" aria-labelledby="so-chart-heading">
|
|
85
|
-
<div class="so-dashboard-section__header">
|
|
86
|
-
<h2 class="so-dashboard-section__title" id="so-chart-heading">Charts</h2>
|
|
87
|
-
</div>
|
|
88
|
-
<%= render "chart", range: @range %>
|
|
89
|
-
</section>
|
|
90
|
-
<% end %>
|
|
91
|
-
|
|
92
|
-
<%# Zone E: Per-queue table %>
|
|
93
|
-
<%= render "queue_table", stats: @stats, range: @range %>
|
|
94
|
-
|
|
95
|
-
<%# Zone F: Stability strip %>
|
|
96
|
-
<% if @stats[:available] && persistence_mode? %>
|
|
97
|
-
<section class="so-stability" data-so-zone="stability" aria-labelledby="so-stability-heading">
|
|
98
|
-
<span class="so-stability__label" id="so-stability-heading">Stability</span>
|
|
99
|
-
<%= stability_badge(@stats) %>
|
|
100
|
-
<span class="so-stability__detail"><%= stability_detail(@stats) %></span>
|
|
101
|
-
<%= link_to "View failures →", events_path(event_type: "job_failed"), class: "so-stability__link" %>
|
|
102
|
-
</section>
|
|
103
|
-
<% end %>
|
|
104
|
-
|
|
105
|
-
<%# Zone G: Recent Events %>
|
|
106
|
-
<% if @stats[:available] && persistence_mode? %>
|
|
107
|
-
<% if defined?(@recent_events) && @recent_events.present? %>
|
|
108
|
-
<div class="so-card">
|
|
109
|
-
<div class="so-card__label">Recent Events</div>
|
|
110
|
-
<table class="so-table so-table--card">
|
|
111
|
-
<caption class="sr-only">Recent queue events</caption>
|
|
112
|
-
<thead>
|
|
113
|
-
<tr>
|
|
114
|
-
<th>Event</th>
|
|
115
|
-
<th>Job Class</th>
|
|
116
|
-
<th>Queue</th>
|
|
117
|
-
<th>Time</th>
|
|
118
|
-
</tr>
|
|
119
|
-
</thead>
|
|
120
|
-
<tbody>
|
|
121
|
-
<% @recent_events.each do |event| %>
|
|
122
|
-
<tr>
|
|
123
|
-
<td><%= event.event_type.humanize %></td>
|
|
124
|
-
<td><%= event.job_class %></td>
|
|
125
|
-
<td><%= event.queue_name %></td>
|
|
126
|
-
<td><%= time_ago_in_words(event.recorded_at) %> ago</td>
|
|
127
|
-
</tr>
|
|
128
|
-
<% end %>
|
|
129
|
-
</tbody>
|
|
130
|
-
</table>
|
|
131
|
-
</div>
|
|
132
|
-
<% else %>
|
|
133
|
-
<div class="so-card">
|
|
134
|
-
<div class="so-card__label">Recent Events</div>
|
|
135
|
-
<p class="so-dashboard-section__meta so-dashboard-section__meta--empty-events">No recent events yet. Queue activity will appear here after jobs are enqueued, performed, or failed.</p>
|
|
136
|
-
</div>
|
|
137
|
-
<% end %>
|
|
138
|
-
<% end %>
|
|
139
|
-
<% end %>
|
|
140
|
-
</div>
|
|
3
|
+
<% if @component == "home" %>
|
|
4
|
+
<%= render "home", health: @health, feed: @feed %>
|
|
5
|
+
<% else %>
|
|
6
|
+
<%= render "queue" %>
|
|
7
|
+
<% end %>
|
|
@@ -38,7 +38,12 @@
|
|
|
38
38
|
<tbody>
|
|
39
39
|
<% @events.each do |event| %>
|
|
40
40
|
<tr>
|
|
41
|
-
<td
|
|
41
|
+
<td>
|
|
42
|
+
<%= link_to event.event_type.humanize, event_path(event.id) %>
|
|
43
|
+
<% if event.correlation_id.present? %>
|
|
44
|
+
<%= link_to "Trace →", trace_path(event.correlation_id), class: "so-trace__link" %>
|
|
45
|
+
<% end %>
|
|
46
|
+
</td>
|
|
42
47
|
<td><%= event.job_class %></td>
|
|
43
48
|
<td><%= event.queue_name %></td>
|
|
44
49
|
<td><%= duration_with_semantic(event.duration, event.event_type) %></td>
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
<% content_for :title, "Trace #{params[:id]}" %>
|
|
2
|
+
|
|
3
|
+
<div class="so-trace">
|
|
4
|
+
<div class="so-back-link">
|
|
5
|
+
<%= link_to "← Back to Events", events_path, class: "so-btn" %>
|
|
6
|
+
</div>
|
|
7
|
+
|
|
8
|
+
<div class="so-content__header">
|
|
9
|
+
<h1>Trace <span class="so-trace__correlation"><%= params[:id] %></span></h1>
|
|
10
|
+
</div>
|
|
11
|
+
|
|
12
|
+
<% if @rows.any? %>
|
|
13
|
+
<p class="so-trace__meta">Showing newest <%= @rows.size %> events (maximum 100)</p>
|
|
14
|
+
<% end %>
|
|
15
|
+
|
|
16
|
+
<% if @unavailable_components.any? %>
|
|
17
|
+
<div class="so-flash so-flash--notice" role="status">
|
|
18
|
+
Some components could not be read: <%= @unavailable_components.map(&:to_s).sort.join(", ") %>. Showing available events only.
|
|
19
|
+
</div>
|
|
20
|
+
<% end %>
|
|
21
|
+
|
|
22
|
+
<% if @rows.any? %>
|
|
23
|
+
<% component_tones = {queue: "info", cache: "success", cable: "warning"} %>
|
|
24
|
+
<table class="so-table so-table--listing so-trace__table">
|
|
25
|
+
<caption class="sr-only">Trace timeline for correlation ID <%= params[:id] %></caption>
|
|
26
|
+
<thead>
|
|
27
|
+
<tr>
|
|
28
|
+
<th class="so-trace__time">Time</th>
|
|
29
|
+
<th class="so-trace__component">Component</th>
|
|
30
|
+
<th class="so-trace__event">Event</th>
|
|
31
|
+
<th class="so-trace__details">Details</th>
|
|
32
|
+
</tr>
|
|
33
|
+
</thead>
|
|
34
|
+
<tbody>
|
|
35
|
+
<% @rows.each do |row| %>
|
|
36
|
+
<tr>
|
|
37
|
+
<td class="so-trace__time"><%= row[:recorded_at].strftime("%Y-%m-%d %H:%M:%S") %></td>
|
|
38
|
+
<td class="so-trace__component">
|
|
39
|
+
<% component_tone = component_tones[row[:component]] || "default" %>
|
|
40
|
+
<span class="so-badge so-badge--pill so-badge--<%= component_tone %>">
|
|
41
|
+
<svg class="so-badge__dot" viewBox="0 0 6 6" aria-hidden="true"><circle r="3" cx="3" cy="3" /></svg>
|
|
42
|
+
<%= row[:component].to_s.capitalize %>
|
|
43
|
+
</span>
|
|
44
|
+
</td>
|
|
45
|
+
<td class="so-trace__event"><%= row[:event_type] %></td>
|
|
46
|
+
<td class="so-trace__details">
|
|
47
|
+
<% if row[:event_type] == "cable_summary" %>
|
|
48
|
+
<%= row[:collapsed_count] %> additional Cable broadcast events collapsed
|
|
49
|
+
<% else %>
|
|
50
|
+
<% case row[:component]
|
|
51
|
+
when :queue %>
|
|
52
|
+
<%= [
|
|
53
|
+
("job=#{row[:job_class]}" if row[:job_class]),
|
|
54
|
+
("queue=#{row[:queue_name]}" if row[:queue_name]),
|
|
55
|
+
("duration=#{format_duration(row[:duration])}" if row[:duration])
|
|
56
|
+
].compact.join(" ") %>
|
|
57
|
+
<% when :cache %>
|
|
58
|
+
<%= [
|
|
59
|
+
("hit=#{row[:hit]}" unless row[:hit].nil?),
|
|
60
|
+
("error=#{row[:error_class]}" if row[:error_class]),
|
|
61
|
+
("duration=#{format_duration(row[:duration])}" if row[:duration])
|
|
62
|
+
].compact.join(" ") %>
|
|
63
|
+
<% when :cable %>
|
|
64
|
+
<%= [
|
|
65
|
+
("channel=#{row[:channel_class]}" if row[:channel_class]),
|
|
66
|
+
("error=#{row[:error_class]}" if row[:error_class]),
|
|
67
|
+
("duration=#{format_duration(row[:duration])}" if row[:duration]),
|
|
68
|
+
("collapsed=#{row[:collapsed_count]}" if row[:collapsed_count])
|
|
69
|
+
].compact.join(" ") %>
|
|
70
|
+
<% end %>
|
|
71
|
+
<% end %>
|
|
72
|
+
</td>
|
|
73
|
+
</tr>
|
|
74
|
+
<% end %>
|
|
75
|
+
</tbody>
|
|
76
|
+
</table>
|
|
77
|
+
<% else %>
|
|
78
|
+
<%= render "solid_observer/shared/empty_state", message: "No events found for correlation ID #{params[:id]}. The trace may have expired, not yet recorded, or belong to an unavailable component." %>
|
|
79
|
+
<% end %>
|
|
80
|
+
</div>
|
data/config/routes.rb
CHANGED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class AddCorrelationIdToCacheEvents < ActiveRecord::Migration[8.0]
|
|
4
|
+
def change
|
|
5
|
+
add_column :solid_observer_cache_events, :correlation_id, :string, limit: 64
|
|
6
|
+
add_index :solid_observer_cache_events, :correlation_id, where: "correlation_id IS NOT NULL"
|
|
7
|
+
end
|
|
8
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class AddCorrelationIdToCableEvents < ActiveRecord::Migration[8.0]
|
|
4
|
+
def change
|
|
5
|
+
add_column :solid_observer_cable_events, :correlation_id, :string, limit: 64
|
|
6
|
+
add_index :solid_observer_cable_events, :correlation_id, where: "correlation_id IS NOT NULL"
|
|
7
|
+
end
|
|
8
|
+
end
|
|
@@ -41,6 +41,9 @@ SolidObserver.configure do |config|
|
|
|
41
41
|
# config.cable_rejection_threshold = 0.05 # Rejection rate → Degraded stability
|
|
42
42
|
# config.cable_backlog_threshold = 0.10 # Backlog ratio threshold
|
|
43
43
|
# config.cable_error_threshold = 0.0 # Error rate threshold
|
|
44
|
+
# === Trace & Health Observability (v0.6.0) ===
|
|
45
|
+
# Health scoring (Services::HealthScore) is auto-computed and config-free.
|
|
46
|
+
# Cross-component trace lookup uses correlation_id across Queue, Cache, and Cable.
|
|
44
47
|
|
|
45
48
|
# Data Retention
|
|
46
49
|
config.event_retention = 30.days # How long to keep event records
|
|
@@ -57,9 +60,10 @@ SolidObserver.configure do |config|
|
|
|
57
60
|
# Sampling (reduce overhead in high-traffic apps)
|
|
58
61
|
config.sampling_rate = 1.0 # 1.0 = 100% (capture all events)
|
|
59
62
|
|
|
60
|
-
# Correlation ID (for
|
|
63
|
+
# Correlation ID (for cross-component tracing & APM integration)
|
|
64
|
+
# Stamped correlation_id powers `rails solid_observer:trace[CORRELATION_ID]` CLI
|
|
65
|
+
# and the Web UI at /solid_observer/traces/:correlation_id.
|
|
61
66
|
# Integrate with your APM tool (Datadog, Sentry, OpenTelemetry, etc.)
|
|
62
|
-
# config.correlation_id_generator = -> {
|
|
63
67
|
# # Example for Datadog APM
|
|
64
68
|
# Datadog::Tracing.active_trace&.id
|
|
65
69
|
#
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidObserver
|
|
4
|
+
module CLI
|
|
5
|
+
class Health < Base
|
|
6
|
+
STATUS_COLORS = {stable: :green, degraded: :yellow, critical: :red}.freeze
|
|
7
|
+
NAME_WIDTH = 8
|
|
8
|
+
UNAVAILABLE_SUFFIX = " (unavailable)"
|
|
9
|
+
|
|
10
|
+
def call
|
|
11
|
+
health = SolidObserver::Services::HealthScore.call
|
|
12
|
+
|
|
13
|
+
print_header
|
|
14
|
+
print_overall(health[:overall])
|
|
15
|
+
print_components(health[:components])
|
|
16
|
+
output("")
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
def print_header
|
|
22
|
+
output("\n🩺 SolidObserver Health", color: :red)
|
|
23
|
+
output("=" * 50, color: :red)
|
|
24
|
+
output("")
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def print_overall(overall)
|
|
28
|
+
output("Overall: #{overall}", color: STATUS_COLORS[overall])
|
|
29
|
+
output("")
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def print_components(components)
|
|
33
|
+
return warning("⚠️ No components enabled") if components.empty?
|
|
34
|
+
|
|
35
|
+
output("Components", color: :green)
|
|
36
|
+
output("")
|
|
37
|
+
components.each { |name, entry| print_component(name, entry[:status], entry[:available]) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Renders the status verbatim: HealthScore owns the worst-of ladder, so an
|
|
41
|
+
# unrecognised status stays uncoloured rather than being coerced to stable.
|
|
42
|
+
# :reek:ControlParameter
|
|
43
|
+
def print_component(name, status, available)
|
|
44
|
+
suffix = available ? "" : UNAVAILABLE_SUFFIX
|
|
45
|
+
output(" #{name.to_s.ljust(NAME_WIDTH)}#{status}#{suffix}", color: STATUS_COLORS[status])
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidObserver
|
|
4
|
+
module CLI
|
|
5
|
+
class Trace < Base
|
|
6
|
+
DETAIL_FORMATTERS = {
|
|
7
|
+
queue: :format_queue_details,
|
|
8
|
+
cache: :format_cache_details,
|
|
9
|
+
cable: :format_cable_details
|
|
10
|
+
}.freeze
|
|
11
|
+
|
|
12
|
+
EVENT_FORMATTERS = {
|
|
13
|
+
"cable_summary" => ->(row) { "and #{row[:collapsed_count]} more cable events" }
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
def call(correlation_id:, limit: nil)
|
|
17
|
+
return output("Trace unavailable in realtime mode.") if SolidObserver.config.realtime_mode?
|
|
18
|
+
|
|
19
|
+
result = SolidObserver::Queries::TraceQuery.new.call(correlation_id: correlation_id, limit: limit)
|
|
20
|
+
return warning("No events found for correlation_id: #{correlation_id}") if (rows = result.rows).empty?
|
|
21
|
+
|
|
22
|
+
print_trace_table(rows)
|
|
23
|
+
print_unavailable_components(result.unavailable_components)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def print_trace_table(rows)
|
|
29
|
+
print_section_header("Trace")
|
|
30
|
+
table(
|
|
31
|
+
headers: ["Time", "Component", "Event", "Details"],
|
|
32
|
+
rows: rows.map { |row| format_cells(row) }
|
|
33
|
+
)
|
|
34
|
+
output("")
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def print_unavailable_components(unavailable)
|
|
38
|
+
unavailable.each { |component| warning("#{component} unavailable") }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def format_cells(row)
|
|
42
|
+
recorded_at, component, event_type = row.values_at(:recorded_at, :component, :event_type)
|
|
43
|
+
[
|
|
44
|
+
format_time(recorded_at),
|
|
45
|
+
component.to_s,
|
|
46
|
+
event_type,
|
|
47
|
+
format_details(row)
|
|
48
|
+
]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def format_details(row)
|
|
52
|
+
event_type, component = row.values_at(:event_type, :component)
|
|
53
|
+
event_formatter = EVENT_FORMATTERS[event_type]
|
|
54
|
+
return event_formatter.call(row) if event_formatter
|
|
55
|
+
|
|
56
|
+
component_formatter = DETAIL_FORMATTERS[component]
|
|
57
|
+
send(component_formatter, row) if component_formatter
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def format_queue_details(row)
|
|
61
|
+
job_class, queue_name, duration = row.values_at(:job_class, :queue_name, :duration)
|
|
62
|
+
[label("job", job_class), label("queue", queue_name), duration_part(duration)].compact.join(" ")
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def format_cache_details(row)
|
|
66
|
+
hit, error_class, duration = row.values_at(:hit, :error_class, :duration)
|
|
67
|
+
[hit_part(hit), error_part(error_class), duration_part(duration)].compact.join(" ")
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def format_cable_details(row)
|
|
71
|
+
channel_class, duration, error_class, collapsed_count = row.values_at(:channel_class, :duration, :error_class, :collapsed_count)
|
|
72
|
+
[label("channel", channel_class), error_part(error_class), duration_part(duration), ("collapsed=#{collapsed_count}" if collapsed_count)].compact.join(" ")
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def label(name, value)
|
|
76
|
+
"#{name}=#{value}" if value
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def error_part(error_class)
|
|
80
|
+
"error=#{error_class}" if error_class
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def hit_part(hit)
|
|
84
|
+
"hit=#{hit}" if [true, false].include?(hit)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def duration_part(duration)
|
|
88
|
+
"duration=#{format_duration(duration)}" if duration
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def format_duration(duration)
|
|
92
|
+
duration ? "#{duration.round(3)}s" : nil
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def format_time(time)
|
|
96
|
+
time.strftime("%Y-%m-%d %H:%M:%S")
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def print_section_header(title)
|
|
100
|
+
output("\n#{title}", color: :red)
|
|
101
|
+
output("=" * 50, color: :red)
|
|
102
|
+
output("")
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
@@ -21,8 +21,7 @@ module SolidObserver
|
|
|
21
21
|
# Observer Settings
|
|
22
22
|
attr_accessor :observe_queue
|
|
23
23
|
|
|
24
|
-
#
|
|
25
|
-
# @note Cable observer is not yet fully implemented
|
|
24
|
+
# Cache & Cable Observer Settings
|
|
26
25
|
attr_accessor :observe_cache,
|
|
27
26
|
:observe_cable,
|
|
28
27
|
:cache_sampling_rate,
|
|
@@ -36,8 +35,6 @@ module SolidObserver
|
|
|
36
35
|
# Retention Settings
|
|
37
36
|
attr_accessor :event_retention
|
|
38
37
|
|
|
39
|
-
# Retention Settings (planned for a future release)
|
|
40
|
-
# @note Metrics cleanup is not yet implemented
|
|
41
38
|
attr_accessor :metrics_retention
|
|
42
39
|
|
|
43
40
|
# Storage Settings
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidObserver
|
|
4
|
+
module Correlated
|
|
5
|
+
def self.included(base)
|
|
6
|
+
base.around_perform do |job, block|
|
|
7
|
+
SolidObserver::Correlated.stamp(job.job_id) { block.call }
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def self.stamp(correlation_id)
|
|
12
|
+
thread_local = Thread.current
|
|
13
|
+
previous = thread_local[:solid_observer_correlation_id]
|
|
14
|
+
thread_local[:solid_observer_correlation_id] = correlation_id if correlation_id.present?
|
|
15
|
+
|
|
16
|
+
yield
|
|
17
|
+
ensure
|
|
18
|
+
thread_local[:solid_observer_correlation_id] = previous
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
ActiveSupport.on_load(:active_job) { include SolidObserver::Correlated }
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidObserver
|
|
4
|
+
module Queries
|
|
5
|
+
class TraceQuery
|
|
6
|
+
DEFAULT_LIMIT = 100
|
|
7
|
+
CABLE_CAP = 50
|
|
8
|
+
|
|
9
|
+
Result = Struct.new(:rows, :unavailable_components, keyword_init: true)
|
|
10
|
+
|
|
11
|
+
COMPONENT_PRIORITY = {queue: 0, cache: 1, cable: 2}.freeze
|
|
12
|
+
|
|
13
|
+
COMPONENT_CONFIG = {
|
|
14
|
+
queue: {
|
|
15
|
+
enabled: :solid_queue_enabled?,
|
|
16
|
+
model: :QueueEvent,
|
|
17
|
+
columns: [:recorded_at, :event_type, :job_class, :queue_name, :duration, :id]
|
|
18
|
+
},
|
|
19
|
+
cache: {
|
|
20
|
+
enabled: :solid_cache_enabled?,
|
|
21
|
+
model: :CacheEvent,
|
|
22
|
+
columns: [:recorded_at, :event_type, :hit, :error_class, :duration, :id]
|
|
23
|
+
},
|
|
24
|
+
cable: {
|
|
25
|
+
enabled: :solid_cable_enabled?,
|
|
26
|
+
model: :CableEvent,
|
|
27
|
+
columns: [:recorded_at, :event_type, :channel_class, :broadcasting_digest, :error_class, :duration, :id]
|
|
28
|
+
}
|
|
29
|
+
}.freeze
|
|
30
|
+
|
|
31
|
+
CONNECTION_ERRORS = [
|
|
32
|
+
ActiveRecord::ConnectionNotEstablished,
|
|
33
|
+
ActiveRecord::StatementInvalid,
|
|
34
|
+
*([PG::ConnectionBad] if defined?(PG::ConnectionBad)),
|
|
35
|
+
*([Mysql2::Error::ConnectionError] if defined?(Mysql2::Error::ConnectionError)),
|
|
36
|
+
*([Trilogy::Error] if defined?(Trilogy::Error)),
|
|
37
|
+
*([SQLite3::CantOpenException] if defined?(SQLite3::CantOpenException))
|
|
38
|
+
].freeze
|
|
39
|
+
|
|
40
|
+
def call(correlation_id:, limit: DEFAULT_LIMIT)
|
|
41
|
+
rows, unavailable = gather_rows(correlation_id)
|
|
42
|
+
effective_limit = limit.to_i
|
|
43
|
+
effective_limit = DEFAULT_LIMIT unless effective_limit.positive?
|
|
44
|
+
Result.new(rows: process_rows(rows, effective_limit), unavailable_components: unavailable)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def gather_rows(correlation_id)
|
|
50
|
+
COMPONENT_CONFIG.each_with_object([[], []]) do |(component, config), (rows, unavailable)|
|
|
51
|
+
next unless SolidObserver.config.public_send(config[:enabled])
|
|
52
|
+
|
|
53
|
+
component_rows = fetch_component_rows(component, config, correlation_id)
|
|
54
|
+
component_rows ? rows.concat(component_rows) : unavailable << component
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def fetch_component_rows(component, config, correlation_id)
|
|
59
|
+
model = SolidObserver.const_get(config[:model])
|
|
60
|
+
return nil unless data_source_available?(model)
|
|
61
|
+
|
|
62
|
+
pluck_rows(model, component, correlation_id)
|
|
63
|
+
rescue *CONNECTION_ERRORS, TypeError
|
|
64
|
+
nil
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def data_source_available?(model)
|
|
68
|
+
table_name = model.table_name.to_s
|
|
69
|
+
table_name.present? && model.connection.data_source_exists?(table_name)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def pluck_rows(model, component, correlation_id)
|
|
73
|
+
columns = COMPONENT_CONFIG[component][:columns]
|
|
74
|
+
model.where(correlation_id: correlation_id)
|
|
75
|
+
.order(:recorded_at)
|
|
76
|
+
.pluck(*columns)
|
|
77
|
+
.map { |values| build_row(component, columns, values) }
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def build_row(component, columns, values)
|
|
81
|
+
row = {component: component}
|
|
82
|
+
columns.each_with_index { |column, index| row[column] = values[index] }
|
|
83
|
+
row
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def process_rows(rows, limit)
|
|
87
|
+
collapsed = collapse_cable_rows(rows)
|
|
88
|
+
capped = apply_cable_cap(collapsed)
|
|
89
|
+
sorted = sort_rows(capped)
|
|
90
|
+
return sorted if capped.size <= limit
|
|
91
|
+
|
|
92
|
+
sort_rows(sorted.last(limit))
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def collapse_cable_rows(rows)
|
|
96
|
+
cable_rows, other_rows = rows.partition { |row| row[:component] == :cable }
|
|
97
|
+
return rows if cable_rows.empty?
|
|
98
|
+
|
|
99
|
+
other_rows + cable_rows.chunk_while(&method(:collapse_match?)).map(&method(:collapse_group))
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def collapse_match?(first_row, second_row)
|
|
103
|
+
return false unless first_row[:event_type] == "broadcast" && second_row[:event_type] == "broadcast"
|
|
104
|
+
|
|
105
|
+
digest = first_row[:broadcasting_digest]
|
|
106
|
+
digest.present? && digest == second_row[:broadcasting_digest]
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def collapse_group(group)
|
|
110
|
+
row = group.first
|
|
111
|
+
size = group.size
|
|
112
|
+
row[:collapsed_count] = size if size > 1
|
|
113
|
+
row.delete(:broadcasting_digest)
|
|
114
|
+
row
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def apply_cable_cap(rows)
|
|
118
|
+
cable_rows, other_rows = rows.partition { |row| row[:component] == :cable }
|
|
119
|
+
total = cable_rows.size
|
|
120
|
+
return rows if total <= CABLE_CAP
|
|
121
|
+
|
|
122
|
+
other_rows + cap_cable_rows(cable_rows, total)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def cap_cable_rows(cable_rows, total)
|
|
126
|
+
keep_count = CABLE_CAP - 1
|
|
127
|
+
[build_summary_row(cable_rows.first(total - keep_count))] + tail_rows(cable_rows, keep_count)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def tail_rows(rows, count)
|
|
131
|
+
rows.last(count)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def build_summary_row(overflow)
|
|
135
|
+
{
|
|
136
|
+
recorded_at: overflow.last[:recorded_at],
|
|
137
|
+
component: :cable,
|
|
138
|
+
event_type: "cable_summary",
|
|
139
|
+
duration: nil,
|
|
140
|
+
channel_class: nil,
|
|
141
|
+
collapsed_count: overflow.size,
|
|
142
|
+
id: -1
|
|
143
|
+
}
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def sort_rows(rows)
|
|
147
|
+
rows.sort_by { |row| [row[:recorded_at], COMPONENT_PRIORITY[row[:component]], row[:id] || 0] }
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|