rails_pulse 0.5.0.pre.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -0
- data/README.md +1 -1
- data/app/controllers/rails_pulse/api/v1/base_controller.rb +4 -1
- data/app/controllers/rails_pulse/api/v1/deployments_controller.rb +17 -1
- data/app/controllers/rails_pulse/api/v1/exceptions_controller.rb +1 -0
- data/app/controllers/rails_pulse/api/v1/insights_controller.rb +62 -0
- data/app/controllers/rails_pulse/api/v1/queries_controller.rb +10 -1
- data/app/controllers/rails_pulse/api/v1/requests_controller.rb +17 -1
- data/app/controllers/rails_pulse/api/v1/routes_controller.rb +6 -1
- data/app/controllers/rails_pulse/dashboard_controller.rb +7 -3
- data/app/jobs/rails_pulse/cloud_health_job.rb +11 -0
- data/app/jobs/rails_pulse/cloud_sync_job.rb +11 -0
- data/app/jobs/rails_pulse/summary_job.rb +13 -0
- data/app/models/rails_pulse/charts/base.rb +3 -12
- data/app/models/rails_pulse/charts/percentile_chart_base.rb +1 -2
- data/app/models/rails_pulse/cleanup_run.rb +32 -0
- data/app/models/rails_pulse/cloud/buffered_batch.rb +55 -0
- data/app/models/rails_pulse/cloud/installation.rb +55 -0
- data/app/models/rails_pulse/dashboard/concerns/attention_classification.rb +104 -0
- data/app/models/rails_pulse/dashboard/concerns/time_range_helper.rb +24 -0
- data/app/models/rails_pulse/dashboard/health_summary.rb +9 -11
- data/app/models/rails_pulse/dashboard/needs_attention.rb +30 -96
- data/app/models/rails_pulse/dashboard/storage_pressure.rb +7 -61
- data/app/models/rails_pulse/dashboard/storage_status.rb +94 -39
- data/app/models/rails_pulse/event.rb +3 -2
- data/app/models/rails_pulse/exceptions/charts/occurrence_volume.rb +1 -3
- data/app/models/rails_pulse/jobs/charts/duration.rb +1 -2
- data/app/models/rails_pulse/jobs/charts/execution_volume.rb +1 -1
- data/app/models/rails_pulse/jobs/charts/failure_rate.rb +1 -2
- data/app/models/rails_pulse/like_pattern.rb +6 -0
- data/app/models/rails_pulse/queries/charts/database_load.rb +1 -3
- data/app/models/rails_pulse/queries/charts/execution_volume.rb +1 -1
- data/app/models/rails_pulse/routes/charts/error_rate.rb +1 -1
- data/app/models/rails_pulse/routes/charts/request_volume.rb +1 -1
- data/app/models/rails_pulse/time_window.rb +15 -0
- data/app/models/rails_pulse/writer_heartbeat.rb +6 -2
- data/app/serializers/rails_pulse/api/v1/deployment_serializer.rb +5 -2
- data/app/serializers/rails_pulse/api/v1/request_serializer.rb +1 -0
- data/app/services/rails_pulse/cloud/batch.rb +83 -0
- data/app/services/rails_pulse/cloud/client.rb +68 -0
- data/app/services/rails_pulse/cloud/combined_metrics.rb +49 -0
- data/app/services/rails_pulse/cloud/health_item.rb +77 -0
- data/app/services/rails_pulse/cloud/path_prefix.rb +50 -0
- data/app/services/rails_pulse/cloud/preview.rb +83 -0
- data/app/services/rails_pulse/cloud/query_shape.rb +195 -0
- data/app/services/rails_pulse/cloud/record_items.rb +60 -0
- data/app/services/rails_pulse/cloud/route_pattern.rb +111 -0
- data/app/services/rails_pulse/cloud/summary_items.rb +188 -0
- data/app/services/rails_pulse/cloud/sync.rb +275 -0
- data/app/services/rails_pulse/config_recommendations.rb +132 -0
- data/app/services/rails_pulse/deployment_comparison.rb +177 -0
- data/app/services/rails_pulse/operations/change_point.rb +1 -1
- data/app/services/rails_pulse/period_insights.rb +131 -0
- data/app/services/rails_pulse/sql_query_normalizer.rb +222 -122
- data/app/views/rails_pulse/dashboard/_storage_panel.html.erb +13 -3
- data/app/views/rails_pulse/storage/show.html.erb +2 -2
- data/config/routes.rb +4 -0
- data/db/rails_pulse_cloud_schema.rb +58 -0
- data/lib/generators/rails_pulse/base_methods.rb +18 -0
- data/lib/generators/rails_pulse/install_cloud_generator.rb +84 -0
- data/lib/generators/rails_pulse/templates/db/rails_pulse_cloud_schema.rb +58 -0
- data/lib/generators/rails_pulse/templates/migrations/install_rails_pulse_cloud_tables.rb +30 -0
- data/lib/generators/rails_pulse/templates/rails_pulse.rb +35 -6
- data/lib/generators/rails_pulse/upgrade_generator.rb +9 -16
- data/lib/rails_pulse/cleanup_service.rb +11 -0
- data/lib/rails_pulse/cli/agent_files/agents.md +13 -5
- data/lib/rails_pulse/cli/agent_files/claude_skill.md +66 -4
- data/lib/rails_pulse/cli/deployments.rb +16 -2
- data/lib/rails_pulse/cli/exceptions.rb +5 -0
- data/lib/rails_pulse/cli/insights.rb +70 -0
- data/lib/rails_pulse/cli/main.rb +4 -0
- data/lib/rails_pulse/cli/queries.rb +5 -0
- data/lib/rails_pulse/cli/requests.rb +9 -1
- data/lib/rails_pulse/cli/routes.rb +7 -0
- data/lib/rails_pulse/configuration.rb +77 -1
- data/lib/rails_pulse/installers/config_updater.rb +9 -2
- data/lib/rails_pulse/mcp/server.rb +3 -1
- data/lib/rails_pulse/mcp/tools/deployments.rb +47 -10
- data/lib/rails_pulse/mcp/tools/errors.rb +11 -2
- data/lib/rails_pulse/mcp/tools/exceptions.rb +16 -2
- data/lib/rails_pulse/mcp/tools/insights.rb +99 -0
- data/lib/rails_pulse/mcp/tools/queries.rb +7 -1
- data/lib/rails_pulse/mcp/tools/routes.rb +12 -1
- data/lib/rails_pulse/tasks/status_reporter.rb +175 -0
- data/lib/rails_pulse/tracker.rb +2 -1
- data/lib/rails_pulse/version.rb +1 -1
- data/lib/tasks/rails_pulse.rake +6 -0
- data/lib/tasks/rails_pulse_tasks.rake +7 -0
- metadata +30 -3
|
@@ -9,6 +9,30 @@ module RailsPulse
|
|
|
9
9
|
|
|
10
10
|
[ @period.days.ago.beginning_of_day, Time.current ]
|
|
11
11
|
end
|
|
12
|
+
|
|
13
|
+
# Summary rows that cover the period exactly once. The same traffic is
|
|
14
|
+
# summarized per hour, day, week and month, so summing across period
|
|
15
|
+
# types would count it once per granularity. A window of 25 hours or
|
|
16
|
+
# less reads hourly rows. A longer one reads daily rows up to the start
|
|
17
|
+
# of today, or to its own end if that comes first, and hourly rows
|
|
18
|
+
# after today's start, because a day is summarized only once it has
|
|
19
|
+
# ended.
|
|
20
|
+
def period_summaries
|
|
21
|
+
start, finish = period_range
|
|
22
|
+
return RailsPulse::Summary.where(period_type: "hour", period_start: start..finish) if hourly_period?(start, finish)
|
|
23
|
+
|
|
24
|
+
today = Time.zone.now.beginning_of_day
|
|
25
|
+
RailsPulse::Summary.where(period_type: "day", period_start: start...[ today, finish ].min)
|
|
26
|
+
.or(RailsPulse::Summary.where(period_type: "hour", period_start: today..finish))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# The dashboard passes the period type TimeRange resolved, so these
|
|
30
|
+
# panels read the same granularity as the cards and charts beside them.
|
|
31
|
+
def hourly_period?(start, finish)
|
|
32
|
+
return @period_type == "hour" if @period_type
|
|
33
|
+
|
|
34
|
+
finish - start <= 25.hours
|
|
35
|
+
end
|
|
12
36
|
end
|
|
13
37
|
end
|
|
14
38
|
end
|
|
@@ -4,11 +4,13 @@ module RailsPulse
|
|
|
4
4
|
include Concerns::ThresholdConstants
|
|
5
5
|
include Concerns::TimeRangeHelper
|
|
6
6
|
|
|
7
|
-
def initialize(disabled_tags: [], show_non_tagged: true, period: 7, window: nil)
|
|
7
|
+
def initialize(disabled_tags: [], show_non_tagged: true, period: 7, period_type: nil, window: nil, storage_pressure: nil)
|
|
8
8
|
@disabled_tags = disabled_tags
|
|
9
9
|
@show_non_tagged = show_non_tagged
|
|
10
10
|
@period = period
|
|
11
|
+
@period_type = period_type
|
|
11
12
|
@window = window
|
|
13
|
+
@storage_pressure = storage_pressure
|
|
12
14
|
@route_thresholds = RailsPulse.configuration.route_thresholds
|
|
13
15
|
@query_thresholds = RailsPulse.configuration.query_thresholds
|
|
14
16
|
@job_thresholds = RailsPulse.configuration.job_thresholds
|
|
@@ -29,11 +31,10 @@ module RailsPulse
|
|
|
29
31
|
private
|
|
30
32
|
|
|
31
33
|
def route_counts
|
|
32
|
-
start, finish = period_range
|
|
33
|
-
|
|
34
34
|
data = RailsPulse::Summary
|
|
35
35
|
.with_tag_filters(@disabled_tags, @show_non_tagged)
|
|
36
|
-
.where(summarizable_type: "RailsPulse::Route"
|
|
36
|
+
.where(summarizable_type: "RailsPulse::Route")
|
|
37
|
+
.merge(period_summaries)
|
|
37
38
|
.group("summarizable_id")
|
|
38
39
|
.select(
|
|
39
40
|
"summarizable_id",
|
|
@@ -62,11 +63,10 @@ module RailsPulse
|
|
|
62
63
|
end
|
|
63
64
|
|
|
64
65
|
def query_counts
|
|
65
|
-
start, finish = period_range
|
|
66
|
-
|
|
67
66
|
data = RailsPulse::Summary
|
|
68
67
|
.with_tag_filters(@disabled_tags, @show_non_tagged)
|
|
69
|
-
.where(summarizable_type: "RailsPulse::Query"
|
|
68
|
+
.where(summarizable_type: "RailsPulse::Query")
|
|
69
|
+
.merge(period_summaries)
|
|
70
70
|
.group("summarizable_id")
|
|
71
71
|
.select(
|
|
72
72
|
"summarizable_id",
|
|
@@ -90,7 +90,7 @@ module RailsPulse
|
|
|
90
90
|
end
|
|
91
91
|
|
|
92
92
|
def storage_counts
|
|
93
|
-
StoragePressure.new.storage_counts
|
|
93
|
+
(@storage_pressure || StoragePressure.new).storage_counts
|
|
94
94
|
end
|
|
95
95
|
|
|
96
96
|
# A process that dropped and has since gone away still counts as
|
|
@@ -152,12 +152,10 @@ module RailsPulse
|
|
|
152
152
|
return nil unless RailsPulse.configuration.track_exceptions
|
|
153
153
|
return nil unless exception_summaries_available?
|
|
154
154
|
|
|
155
|
-
start, finish = period_range
|
|
156
|
-
|
|
157
155
|
counts = RailsPulse::Summary
|
|
158
156
|
.for_exceptions
|
|
159
157
|
.where.not(summarizable_id: 0)
|
|
160
|
-
.
|
|
158
|
+
.merge(period_summaries)
|
|
161
159
|
.group(:summarizable_id)
|
|
162
160
|
.sum(:count)
|
|
163
161
|
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
module RailsPulse
|
|
2
2
|
module Dashboard
|
|
3
3
|
class NeedsAttention
|
|
4
|
-
include Concerns::
|
|
4
|
+
include Concerns::AttentionClassification
|
|
5
5
|
include Concerns::TimeRangeHelper
|
|
6
6
|
|
|
7
|
-
def initialize(disabled_tags: [], show_non_tagged: true, period: 7, window: nil)
|
|
7
|
+
def initialize(disabled_tags: [], show_non_tagged: true, period: 7, period_type: nil, window: nil, storage_pressure: nil)
|
|
8
8
|
@disabled_tags = disabled_tags
|
|
9
9
|
@show_non_tagged = show_non_tagged
|
|
10
10
|
@period = period
|
|
11
|
+
@period_type = period_type
|
|
11
12
|
@window = window
|
|
13
|
+
@storage_pressure = storage_pressure
|
|
12
14
|
@route_thresholds = RailsPulse.configuration.route_thresholds
|
|
13
15
|
@query_thresholds = RailsPulse.configuration.query_thresholds
|
|
14
16
|
@job_thresholds = RailsPulse.configuration.job_thresholds
|
|
@@ -40,7 +42,18 @@ module RailsPulse
|
|
|
40
42
|
private
|
|
41
43
|
|
|
42
44
|
def storage_pressure_items
|
|
43
|
-
StoragePressure.new.pressure_items
|
|
45
|
+
(@storage_pressure || StoragePressure.new).pressure_items
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# The dashboard window is user-selected, so the error-rate reason names
|
|
49
|
+
# the window's real length rather than assuming a week.
|
|
50
|
+
def span_phrase
|
|
51
|
+
start, finish = period_range
|
|
52
|
+
hours = ((finish - start) / 1.hour).round
|
|
53
|
+
return "in the last #{hours} hour#{"s" unless hours == 1}" if hours <= 25
|
|
54
|
+
|
|
55
|
+
days = (hours / 24.0).round
|
|
56
|
+
"in the last #{days} day#{"s" unless days == 1}"
|
|
44
57
|
end
|
|
45
58
|
|
|
46
59
|
def url_helpers
|
|
@@ -48,12 +61,11 @@ module RailsPulse
|
|
|
48
61
|
end
|
|
49
62
|
|
|
50
63
|
def route_items
|
|
51
|
-
start, finish = period_range
|
|
52
|
-
|
|
53
64
|
route_data = RailsPulse::Summary
|
|
54
65
|
.with_tag_filters(@disabled_tags, @show_non_tagged)
|
|
55
66
|
.joins("INNER JOIN rails_pulse_routes ON rails_pulse_routes.id = rails_pulse_summaries.summarizable_id")
|
|
56
|
-
.where(summarizable_type: "RailsPulse::Route"
|
|
67
|
+
.where(summarizable_type: "RailsPulse::Route")
|
|
68
|
+
.merge(period_summaries)
|
|
57
69
|
.group("rails_pulse_summaries.summarizable_id, rails_pulse_routes.path")
|
|
58
70
|
.select(
|
|
59
71
|
"rails_pulse_summaries.summarizable_id as route_id",
|
|
@@ -70,7 +82,7 @@ module RailsPulse
|
|
|
70
82
|
errors = record.total_errors.to_i
|
|
71
83
|
error_rate = total > 0 ? (errors * 100.0 / total).round(1) : 0.0
|
|
72
84
|
|
|
73
|
-
severity, reason, metric, metric_sub, sort_score = classify_route(p95, total, errors, error_rate)
|
|
85
|
+
severity, reason, metric, metric_sub, sort_score = classify_route(p95, total, errors, error_rate, span_phrase)
|
|
74
86
|
next unless severity
|
|
75
87
|
|
|
76
88
|
items << {
|
|
@@ -88,45 +100,12 @@ module RailsPulse
|
|
|
88
100
|
items
|
|
89
101
|
end
|
|
90
102
|
|
|
91
|
-
def classify_route(p95, total, errors, error_rate)
|
|
92
|
-
if p95 >= @route_thresholds[:critical] || error_rate >= CRITICAL_ERROR_RATE
|
|
93
|
-
if error_rate >= CRITICAL_ERROR_RATE
|
|
94
|
-
[ :critical,
|
|
95
|
-
"#{error_rate}% error rate · #{total} requests this week",
|
|
96
|
-
"#{errors} errors",
|
|
97
|
-
"P95 #{p95.round(0).to_i}ms",
|
|
98
|
-
errors * total.to_f ]
|
|
99
|
-
else
|
|
100
|
-
[ :critical,
|
|
101
|
-
"#{p95.round(0).to_i}ms P95 · exceeds #{@route_thresholds[:critical]}ms threshold",
|
|
102
|
-
"#{p95.round(0).to_i}ms P95",
|
|
103
|
-
"#{total} requests",
|
|
104
|
-
p95 ]
|
|
105
|
-
end
|
|
106
|
-
elsif p95 >= @route_thresholds[:slow] || error_rate >= WARNING_ERROR_RATE
|
|
107
|
-
if error_rate >= WARNING_ERROR_RATE && p95 < @route_thresholds[:slow]
|
|
108
|
-
[ :warning,
|
|
109
|
-
"#{error_rate}% error rate · #{total} requests this week",
|
|
110
|
-
"#{errors} errors",
|
|
111
|
-
"P95 #{p95.round(0).to_i}ms",
|
|
112
|
-
errors * total.to_f ]
|
|
113
|
-
else
|
|
114
|
-
[ :warning,
|
|
115
|
-
"#{p95.round(0).to_i}ms P95 · #{error_rate > 0 ? "#{error_rate}% error rate" : "above slow threshold"}",
|
|
116
|
-
"#{p95.round(0).to_i}ms P95",
|
|
117
|
-
"#{total} requests",
|
|
118
|
-
p95 ]
|
|
119
|
-
end
|
|
120
|
-
end
|
|
121
|
-
end
|
|
122
|
-
|
|
123
103
|
def query_items
|
|
124
|
-
start, finish = period_range
|
|
125
|
-
|
|
126
104
|
query_data = RailsPulse::Summary
|
|
127
105
|
.with_tag_filters(@disabled_tags, @show_non_tagged)
|
|
128
106
|
.joins("INNER JOIN rails_pulse_queries ON rails_pulse_queries.id = rails_pulse_summaries.summarizable_id")
|
|
129
|
-
.where(summarizable_type: "RailsPulse::Query"
|
|
107
|
+
.where(summarizable_type: "RailsPulse::Query")
|
|
108
|
+
.merge(period_summaries)
|
|
130
109
|
.group("rails_pulse_summaries.summarizable_id, rails_pulse_queries.normalized_sql")
|
|
131
110
|
.select(
|
|
132
111
|
"rails_pulse_summaries.summarizable_id as query_id",
|
|
@@ -142,26 +121,19 @@ module RailsPulse
|
|
|
142
121
|
p95 = record.p95_duration.to_f
|
|
143
122
|
count = record.total_count.to_i
|
|
144
123
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
reason = "#{p95.round(0).to_i}ms P95 · exceeds #{@query_thresholds[:critical]}ms threshold"
|
|
148
|
-
elsif p95 >= @query_thresholds[:slow]
|
|
149
|
-
severity = :warning
|
|
150
|
-
reason = "#{p95.round(0).to_i}ms P95 · above #{@query_thresholds[:slow]}ms slow threshold"
|
|
151
|
-
else
|
|
152
|
-
next
|
|
153
|
-
end
|
|
124
|
+
severity, reason, metric, metric_sub, sort_score = classify_query(p95, count)
|
|
125
|
+
next unless severity
|
|
154
126
|
|
|
155
127
|
classified_ids << record.query_id
|
|
156
128
|
items << {
|
|
157
129
|
type: "QUERY",
|
|
158
130
|
name: truncate_sql(record.normalized_sql),
|
|
159
131
|
reason: reason,
|
|
160
|
-
metric:
|
|
161
|
-
metric_sub:
|
|
132
|
+
metric: metric,
|
|
133
|
+
metric_sub: metric_sub,
|
|
162
134
|
link: url_helpers.query_path(record.query_id),
|
|
163
135
|
severity: severity,
|
|
164
|
-
sort_score:
|
|
136
|
+
sort_score: sort_score,
|
|
165
137
|
monospace: true
|
|
166
138
|
}
|
|
167
139
|
end
|
|
@@ -199,7 +171,9 @@ module RailsPulse
|
|
|
199
171
|
failure_rate = job.failure_rate
|
|
200
172
|
p95 = job.p95_duration.to_f
|
|
201
173
|
|
|
202
|
-
severity, reason, metric, metric_sub, sort_score = classify_job(
|
|
174
|
+
severity, reason, metric, metric_sub, sort_score = classify_job(
|
|
175
|
+
failure_rate, p95, job.runs_count, job.failures_count, job.queue_name.presence || "default"
|
|
176
|
+
)
|
|
203
177
|
next unless severity
|
|
204
178
|
|
|
205
179
|
items << {
|
|
@@ -217,38 +191,6 @@ module RailsPulse
|
|
|
217
191
|
items
|
|
218
192
|
end
|
|
219
193
|
|
|
220
|
-
def classify_job(job, failure_rate, p95)
|
|
221
|
-
if failure_rate >= CRITICAL_JOB_FAILURE_RATE || p95 >= @job_thresholds[:critical]
|
|
222
|
-
if failure_rate >= CRITICAL_JOB_FAILURE_RATE
|
|
223
|
-
[ :critical,
|
|
224
|
-
"#{job.queue_name.presence || "default"} queue · #{failure_rate}% failure rate",
|
|
225
|
-
"#{job.failures_count} / #{job.runs_count} failed",
|
|
226
|
-
"P95 #{p95.round(0).to_i}ms",
|
|
227
|
-
job.failures_count * job.runs_count.to_f ]
|
|
228
|
-
else
|
|
229
|
-
[ :critical,
|
|
230
|
-
"P95 #{p95.round(0).to_i}ms · exceeds #{@job_thresholds[:critical]}ms threshold",
|
|
231
|
-
"#{p95.round(0).to_i}ms P95",
|
|
232
|
-
"#{job.runs_count} runs",
|
|
233
|
-
p95 ]
|
|
234
|
-
end
|
|
235
|
-
elsif failure_rate >= WARNING_JOB_FAILURE_RATE || p95 >= @job_thresholds[:slow]
|
|
236
|
-
if failure_rate >= WARNING_JOB_FAILURE_RATE
|
|
237
|
-
[ :warning,
|
|
238
|
-
"#{job.queue_name.presence || "default"} queue · #{failure_rate}% failure rate",
|
|
239
|
-
"#{job.failures_count} / #{job.runs_count} failed",
|
|
240
|
-
"P95 #{p95.round(0).to_i}ms",
|
|
241
|
-
job.failures_count * job.runs_count.to_f ]
|
|
242
|
-
else
|
|
243
|
-
[ :warning,
|
|
244
|
-
"P95 #{p95.round(0).to_i}ms · above slow threshold",
|
|
245
|
-
"#{p95.round(0).to_i}ms P95",
|
|
246
|
-
"#{job.runs_count} runs",
|
|
247
|
-
p95 ]
|
|
248
|
-
end
|
|
249
|
-
end
|
|
250
|
-
end
|
|
251
|
-
|
|
252
194
|
# Exception groups that fired often enough over the period to be worth
|
|
253
195
|
# looking at. Read from summaries, not from ExceptionGroup#occurrence_count
|
|
254
196
|
# — that is a lifetime counter, so a group that was noisy last year and is
|
|
@@ -261,12 +203,10 @@ module RailsPulse
|
|
|
261
203
|
return [] unless RailsPulse.configuration.track_exceptions
|
|
262
204
|
return [] unless exception_tables_available?
|
|
263
205
|
|
|
264
|
-
start, finish = period_range
|
|
265
|
-
|
|
266
206
|
counts = RailsPulse::Summary
|
|
267
207
|
.for_exceptions
|
|
268
208
|
.where.not(summarizable_id: 0)
|
|
269
|
-
.
|
|
209
|
+
.merge(period_summaries)
|
|
270
210
|
.group(:summarizable_id)
|
|
271
211
|
.sum(:count)
|
|
272
212
|
|
|
@@ -319,12 +259,6 @@ module RailsPulse
|
|
|
319
259
|
rescue ActiveRecord::ActiveRecordError
|
|
320
260
|
false
|
|
321
261
|
end
|
|
322
|
-
|
|
323
|
-
def truncate_sql(sql)
|
|
324
|
-
return "" if sql.blank?
|
|
325
|
-
cleaned = sql.gsub(/\s+/, " ").strip
|
|
326
|
-
cleaned.length > 80 ? "#{cleaned[0..79]}..." : cleaned
|
|
327
|
-
end
|
|
328
262
|
end
|
|
329
263
|
end
|
|
330
264
|
end
|
|
@@ -8,8 +8,10 @@ module RailsPulse
|
|
|
8
8
|
@config = RailsPulse.configuration
|
|
9
9
|
end
|
|
10
10
|
|
|
11
|
+
# Memoized: computing the items costs several aggregate queries (one a
|
|
12
|
+
# raw-request count), and one dashboard render asks for them repeatedly.
|
|
11
13
|
def pressure_items
|
|
12
|
-
|
|
14
|
+
@pressure_items ||= summary_staleness_items + sub_hour_retention_items + writer_drop_items
|
|
13
15
|
end
|
|
14
16
|
|
|
15
17
|
# Dropped requests get their own Tracking badge, not this one
|
|
@@ -47,43 +49,7 @@ module RailsPulse
|
|
|
47
49
|
end
|
|
48
50
|
end
|
|
49
51
|
|
|
50
|
-
# Signal B —
|
|
51
|
-
def stuck_records_items
|
|
52
|
-
return [] unless @config.archiving_enabled
|
|
53
|
-
return [] unless @config.full_retention_period
|
|
54
|
-
|
|
55
|
-
oldest_start = oldest_overall_request_summary_start
|
|
56
|
-
return [] if oldest_start.nil?
|
|
57
|
-
|
|
58
|
-
stuck_count = RailsPulse::Request
|
|
59
|
-
.where("occurred_at < ?", @config.full_retention_period.ago)
|
|
60
|
-
.where("occurred_at < ?", oldest_start)
|
|
61
|
-
.count
|
|
62
|
-
|
|
63
|
-
return [] if stuck_count.zero?
|
|
64
|
-
|
|
65
|
-
[ {
|
|
66
|
-
type: "STORAGE",
|
|
67
|
-
name: "Storage pressure",
|
|
68
|
-
reason: "#{stuck_count} #{"request".pluralize(stuck_count)} past the retention window cannot be cleaned up — summarize first",
|
|
69
|
-
metric: "#{stuck_count} stuck",
|
|
70
|
-
metric_sub: "older than #{humanize_duration(@config.full_retention_period)}",
|
|
71
|
-
link: "#",
|
|
72
|
-
severity: :critical,
|
|
73
|
-
sort_score: stuck_count.to_f,
|
|
74
|
-
popover_title: "Cleanup is blocked on unsummarised records",
|
|
75
|
-
popover_body: "#{stuck_count} #{"request".pluralize(stuck_count)} are older than your retention period " \
|
|
76
|
-
"(#{humanize_duration(@config.full_retention_period)}) but predate any hourly summary record. " \
|
|
77
|
-
"CleanupService will not delete records from a time period until that period has been summarised — " \
|
|
78
|
-
"otherwise those data points would be permanently lost from dashboard charts.<br><br>" \
|
|
79
|
-
"<strong>How to fix:</strong> Run <code>RailsPulse::SummaryJob</code> to summarise the historical periods. " \
|
|
80
|
-
"If the job is not yet scheduled, run the backfill task to summarise all historical data:<br><br>" \
|
|
81
|
-
"<code>rails rails_pulse:backfill_summaries</code><br><br>" \
|
|
82
|
-
"Once the affected periods are summarised, the next cleanup run will delete these records automatically."
|
|
83
|
-
} ]
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
# Signal C — Retention period shorter than 1-hour cleanup minimum
|
|
52
|
+
# Signal B — Retention period shorter than 1-hour cleanup minimum
|
|
87
53
|
def sub_hour_retention_items
|
|
88
54
|
return [] unless @config.full_retention_period
|
|
89
55
|
return [] unless @config.full_retention_period < 1.hour
|
|
@@ -108,7 +74,7 @@ module RailsPulse
|
|
|
108
74
|
} ]
|
|
109
75
|
end
|
|
110
76
|
|
|
111
|
-
# Signal
|
|
77
|
+
# Signal C — the background writer is discarding requests
|
|
112
78
|
def writer_drop_items
|
|
113
79
|
return [] unless RailsPulse::Event.table_available?
|
|
114
80
|
|
|
@@ -148,8 +114,8 @@ module RailsPulse
|
|
|
148
114
|
severity: severity,
|
|
149
115
|
sort_score: severity == :critical ? Float::INFINITY : 1.0,
|
|
150
116
|
popover_title: "Summary job is not running",
|
|
151
|
-
popover_body: "Rails Pulse summarises raw request data into hourly aggregates
|
|
152
|
-
"
|
|
117
|
+
popover_body: "Rails Pulse summarises raw request data into hourly aggregates for the dashboard's charts. " \
|
|
118
|
+
"Count-based cleanup only trims records from periods that have been summarised, so until the job catches up tables can grow past <code>max_table_records</code>.<br><br>" \
|
|
153
119
|
"<strong>How to fix:</strong> Schedule <code>RailsPulse::SummaryJob</code> to run every hour in your job scheduler " \
|
|
154
120
|
"(Sidekiq-Cron, GoodJob, Solid Queue, etc.). Once it has run, cleanup will resume automatically on its next execution.<br><br>" \
|
|
155
121
|
"To backfill any historical gaps, run: <code>rails rails_pulse:backfill_summaries</code>"
|
|
@@ -161,26 +127,6 @@ module RailsPulse
|
|
|
161
127
|
.where(summarizable_type: "RailsPulse::Request", summarizable_id: 0, period_type: "hour")
|
|
162
128
|
.maximum(:period_end)
|
|
163
129
|
end
|
|
164
|
-
|
|
165
|
-
def oldest_overall_request_summary_start
|
|
166
|
-
RailsPulse::Summary
|
|
167
|
-
.where(summarizable_type: "RailsPulse::Request", summarizable_id: 0, period_type: "hour")
|
|
168
|
-
.minimum(:period_start)
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
def humanize_duration(duration)
|
|
172
|
-
total_seconds = duration.to_i
|
|
173
|
-
if total_seconds >= 86_400
|
|
174
|
-
days = total_seconds / 86_400
|
|
175
|
-
"#{days} #{"day".pluralize(days)}"
|
|
176
|
-
elsif total_seconds >= 3_600
|
|
177
|
-
hours = total_seconds / 3_600
|
|
178
|
-
"#{hours} #{"hour".pluralize(hours)}"
|
|
179
|
-
else
|
|
180
|
-
minutes = total_seconds / 60
|
|
181
|
-
"#{minutes} #{"minute".pluralize(minutes)}"
|
|
182
|
-
end
|
|
183
|
-
end
|
|
184
130
|
end
|
|
185
131
|
end
|
|
186
132
|
end
|
|
@@ -34,7 +34,7 @@ module RailsPulse
|
|
|
34
34
|
name: :rails_pulse_summaries,
|
|
35
35
|
model: "RailsPulse::Summary",
|
|
36
36
|
label: "Summaries",
|
|
37
|
-
description: "Hourly and daily aggregates used by charts. Hourly rows
|
|
37
|
+
description: "Hourly and daily aggregates used by charts. Hourly rows are cleaned up after 7 days by default (hourly_summary_retention)",
|
|
38
38
|
time_column: :period_start
|
|
39
39
|
},
|
|
40
40
|
{
|
|
@@ -78,42 +78,43 @@ module RailsPulse
|
|
|
78
78
|
label: "Events",
|
|
79
79
|
description: "What Pulse noticed: writer heartbeats every minute, pruned after a day; any other kind is kept for event_retention_period",
|
|
80
80
|
time_column: :occurred_at
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
name: :rails_pulse_cloud_batches,
|
|
84
|
+
model: "RailsPulse::Cloud::BufferedBatch",
|
|
85
|
+
label: "Cloud buffer",
|
|
86
|
+
description: "Batches waiting to be sent to Rails Pulse Cloud. Held for at most 7 days or 50 MB",
|
|
87
|
+
time_column: :created_at
|
|
81
88
|
}
|
|
82
89
|
].freeze
|
|
83
90
|
|
|
84
|
-
#
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
return entry[:bytes] if entry && entry[:measured_at] > SIZE_CACHE_TTL.ago
|
|
97
|
-
|
|
98
|
-
bytes = yield
|
|
99
|
-
@size_cache[table_name] = { bytes: bytes, measured_at: Time.current }
|
|
100
|
-
bytes
|
|
101
|
-
end
|
|
102
|
-
end
|
|
91
|
+
# The Storage page counts every row exactly: one unindexed COUNT/MIN/MAX
|
|
92
|
+
# scan per table, which takes tens of seconds on large installs. The
|
|
93
|
+
# dashboard only shows headline counts and fill bars, so with
|
|
94
|
+
# estimated: true it reads the catalog's row estimates instead
|
|
95
|
+
# (pg_class.reltuples, information_schema TABLE_ROWS). SQLite keeps no
|
|
96
|
+
# such estimate, and dbstat walks the whole file, so there the dashboard
|
|
97
|
+
# measures no tables at all.
|
|
98
|
+
def initialize(estimated: false, storage_pressure: nil)
|
|
99
|
+
@config = RailsPulse.configuration
|
|
100
|
+
@pressure = storage_pressure || StoragePressure.new
|
|
101
|
+
@estimated = estimated
|
|
102
|
+
end
|
|
103
103
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
end
|
|
104
|
+
def estimated?
|
|
105
|
+
@estimated
|
|
107
106
|
end
|
|
108
107
|
|
|
109
|
-
def
|
|
110
|
-
@
|
|
111
|
-
@pressure = StoragePressure.new
|
|
112
|
-
@cached_sizes = cached_sizes
|
|
108
|
+
def counts_available?
|
|
109
|
+
!(@estimated && sqlite?)
|
|
113
110
|
end
|
|
114
111
|
|
|
115
112
|
def tables
|
|
116
|
-
@tables ||=
|
|
113
|
+
@tables ||= if counts_available?
|
|
114
|
+
with_current_mysql_statistics { TABLE_CATALOG.filter_map { |definition| build_table(definition) } }
|
|
115
|
+
else
|
|
116
|
+
[]
|
|
117
|
+
end
|
|
117
118
|
end
|
|
118
119
|
|
|
119
120
|
def pressure_items
|
|
@@ -198,12 +199,37 @@ module RailsPulse
|
|
|
198
199
|
RailsPulse::ApplicationRecord.connection
|
|
199
200
|
end
|
|
200
201
|
|
|
202
|
+
# MySQL 8 caches information_schema's TABLE_ROWS, DATA_LENGTH and
|
|
203
|
+
# INDEX_LENGTH for information_schema_stats_expiry seconds, a day by
|
|
204
|
+
# default, so a table that grew after its first read reports its old
|
|
205
|
+
# size. An expiry of 0 reads current statistics; a SET_VAR hint does
|
|
206
|
+
# not reach information_schema, so it is set on the session and put
|
|
207
|
+
# back afterwards because the connection returns to the pool. MariaDB
|
|
208
|
+
# has no such variable and does not cache, so the block runs as is.
|
|
209
|
+
def with_current_mysql_statistics
|
|
210
|
+
previous = session_statistics_expiry
|
|
211
|
+
return yield if previous.nil?
|
|
212
|
+
|
|
213
|
+
connection.execute("SET SESSION information_schema_stats_expiry = 0")
|
|
214
|
+
yield
|
|
215
|
+
ensure
|
|
216
|
+
connection.execute("SET SESSION information_schema_stats_expiry = #{previous.to_i}") unless previous.nil?
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def session_statistics_expiry
|
|
220
|
+
return unless mysql?
|
|
221
|
+
|
|
222
|
+
connection.select_value("SELECT @@SESSION.information_schema_stats_expiry")
|
|
223
|
+
rescue ActiveRecord::StatementInvalid
|
|
224
|
+
nil
|
|
225
|
+
end
|
|
226
|
+
|
|
201
227
|
def build_table(definition)
|
|
202
228
|
model = definition[:model].safe_constantize
|
|
203
229
|
return unless model
|
|
204
230
|
return unless connection.table_exists?(definition[:name])
|
|
205
231
|
|
|
206
|
-
stats =
|
|
232
|
+
stats = measured_counts(model, definition)
|
|
207
233
|
limit = @config.max_table_records&.[](definition[:name])
|
|
208
234
|
count = stats[:count]
|
|
209
235
|
percent = limit.to_i.positive? ? ((count.to_f / limit) * 100).round(1) : nil
|
|
@@ -223,7 +249,7 @@ module RailsPulse
|
|
|
223
249
|
newest_at: stats[:newest_at],
|
|
224
250
|
recent_count: stats[:recent_count],
|
|
225
251
|
runway_label: runway_label(count, limit, stats[:recent_count]),
|
|
226
|
-
bytes:
|
|
252
|
+
bytes: table_bytes(definition[:name]),
|
|
227
253
|
history_label: history_label(stats[:oldest_at], stats[:newest_at])
|
|
228
254
|
}
|
|
229
255
|
rescue => error
|
|
@@ -247,6 +273,33 @@ module RailsPulse
|
|
|
247
273
|
}
|
|
248
274
|
end
|
|
249
275
|
|
|
276
|
+
def measured_counts(model, definition)
|
|
277
|
+
return fetch_counts(model, definition) unless @estimated
|
|
278
|
+
|
|
279
|
+
{ count: estimated_count(definition[:name]), oldest_at: nil, newest_at: nil, recent_count: 0 }
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def estimated_count(table_name)
|
|
283
|
+
if connection.adapter_name.downcase.include?("postgres")
|
|
284
|
+
estimate = connection.select_value(
|
|
285
|
+
"SELECT reltuples::bigint FROM pg_class WHERE oid = to_regclass(#{connection.quote(table_name.to_s)})"
|
|
286
|
+
).to_i
|
|
287
|
+
# -1 until the table's first ANALYZE; autovacuum runs one after
|
|
288
|
+
# about 50 inserts, so a table without an estimate is small enough
|
|
289
|
+
# to count.
|
|
290
|
+
estimate.negative? ? connection.select_value("SELECT COUNT(*) FROM #{connection.quote_table_name(table_name)}").to_i : estimate
|
|
291
|
+
else
|
|
292
|
+
connection.select_value(
|
|
293
|
+
RailsPulse::ApplicationRecord.sanitize_sql_array([ <<~SQL, table_name.to_s ])
|
|
294
|
+
SELECT TABLE_ROWS
|
|
295
|
+
FROM information_schema.TABLES
|
|
296
|
+
WHERE TABLE_SCHEMA = DATABASE()
|
|
297
|
+
AND TABLE_NAME = ?
|
|
298
|
+
SQL
|
|
299
|
+
).to_i
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
|
|
250
303
|
def fetch_counts(model, definition)
|
|
251
304
|
quoted_table = connection.quote_table_name(definition[:name])
|
|
252
305
|
time_column = definition[:time_column]
|
|
@@ -351,12 +404,6 @@ module RailsPulse
|
|
|
351
404
|
row[key] || row[key.to_s] || row[key.to_s.upcase] || row[key.to_sym]
|
|
352
405
|
end
|
|
353
406
|
|
|
354
|
-
def measured_table_bytes(table_name)
|
|
355
|
-
return table_bytes(table_name) unless @cached_sizes
|
|
356
|
-
|
|
357
|
-
self.class.cached_table_bytes(table_name) { table_bytes(table_name) }
|
|
358
|
-
end
|
|
359
|
-
|
|
360
407
|
def table_bytes(table_name)
|
|
361
408
|
adapter = connection.adapter_name.downcase
|
|
362
409
|
|
|
@@ -364,7 +411,7 @@ module RailsPulse
|
|
|
364
411
|
connection.select_value(
|
|
365
412
|
"SELECT pg_total_relation_size(#{connection.quote(table_name.to_s)})"
|
|
366
413
|
).to_i
|
|
367
|
-
elsif
|
|
414
|
+
elsif mysql?
|
|
368
415
|
connection.select_value(
|
|
369
416
|
RailsPulse::ApplicationRecord.sanitize_sql_array([ <<~SQL, table_name.to_s ])
|
|
370
417
|
SELECT DATA_LENGTH + INDEX_LENGTH
|
|
@@ -410,11 +457,19 @@ module RailsPulse
|
|
|
410
457
|
nil
|
|
411
458
|
end
|
|
412
459
|
|
|
460
|
+
def mysql?
|
|
461
|
+
connection.adapter_name.downcase.match?(/mysql|trilogy/)
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
def sqlite?
|
|
465
|
+
connection.adapter_name.downcase.include?("sqlite")
|
|
466
|
+
end
|
|
467
|
+
|
|
413
468
|
def adapter_label
|
|
414
469
|
name = connection.adapter_name
|
|
415
470
|
case name.downcase
|
|
416
471
|
when /postgres/ then "PostgreSQL"
|
|
417
|
-
when /mysql/ then "MySQL"
|
|
472
|
+
when /mysql|trilogy/ then "MySQL"
|
|
418
473
|
when /sqlite/ then "SQLite"
|
|
419
474
|
else name
|
|
420
475
|
end
|
|
@@ -443,7 +498,7 @@ module RailsPulse
|
|
|
443
498
|
"File size of the dedicated Rails Pulse SQLite database."
|
|
444
499
|
elsif adapter.include?("postgres")
|
|
445
500
|
"PostgreSQL relation size including indexes (pg_total_relation_size)."
|
|
446
|
-
elsif
|
|
501
|
+
elsif mysql?
|
|
447
502
|
"MySQL data + index length from information_schema."
|
|
448
503
|
end
|
|
449
504
|
end
|
|
@@ -3,8 +3,9 @@ module RailsPulse
|
|
|
3
3
|
# outcome or sample, tagged by `kind`; `subject` names what it is about,
|
|
4
4
|
# `value` carries its number and `metadata` the kind-specific detail as JSON.
|
|
5
5
|
#
|
|
6
|
-
#
|
|
7
|
-
# table is generic so a new kind needs no
|
|
6
|
+
# Two kinds are written today: writer_heartbeat (see WriterHeartbeat) and
|
|
7
|
+
# cleanup_run (see CleanupRun). The table is generic so a new kind needs no
|
|
8
|
+
# migration. Decision 0019.
|
|
8
9
|
#
|
|
9
10
|
# CleanupService deletes rows older than config.event_retention_period,
|
|
10
11
|
# except kinds listed in config.event_retention_exempt_kinds (rows a writer
|
|
@@ -41,10 +41,8 @@ module RailsPulse
|
|
|
41
41
|
# answer "zero" rather than "no data" — leaving the gap unfilled would
|
|
42
42
|
# draw a chart that skips quiet days and misrepresents the trend.
|
|
43
43
|
def pad_with_zeros(raw)
|
|
44
|
-
step = @period_type.to_s == "hour" ? 3600 : 86_400
|
|
45
|
-
|
|
46
44
|
{}.tap do |padded|
|
|
47
|
-
|
|
45
|
+
@window.bucket_timestamps(@period_type).each do |timestamp|
|
|
48
46
|
padded[timestamp] = raw[timestamp] || 0
|
|
49
47
|
end
|
|
50
48
|
end
|
|
@@ -31,9 +31,8 @@ module RailsPulse
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
# Convert to final values (weighted averages) and pad missing data
|
|
34
|
-
step = time_step
|
|
35
34
|
daily_data = {}
|
|
36
|
-
|
|
35
|
+
@window.bucket_timestamps(@period_type).each do |timestamp|
|
|
37
36
|
if raw_data[timestamp]
|
|
38
37
|
count = raw_data[timestamp][:total_count]
|
|
39
38
|
daily_data[timestamp] = {
|