rails_pulse 0.5.0.pre.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +27 -0
  3. data/README.md +1 -1
  4. data/app/controllers/rails_pulse/api/v1/base_controller.rb +4 -1
  5. data/app/controllers/rails_pulse/api/v1/deployments_controller.rb +17 -1
  6. data/app/controllers/rails_pulse/api/v1/exceptions_controller.rb +1 -0
  7. data/app/controllers/rails_pulse/api/v1/insights_controller.rb +62 -0
  8. data/app/controllers/rails_pulse/api/v1/queries_controller.rb +10 -1
  9. data/app/controllers/rails_pulse/api/v1/requests_controller.rb +17 -1
  10. data/app/controllers/rails_pulse/api/v1/routes_controller.rb +6 -1
  11. data/app/controllers/rails_pulse/dashboard_controller.rb +7 -3
  12. data/app/jobs/rails_pulse/cloud_health_job.rb +11 -0
  13. data/app/jobs/rails_pulse/cloud_sync_job.rb +11 -0
  14. data/app/jobs/rails_pulse/summary_job.rb +13 -0
  15. data/app/models/rails_pulse/charts/base.rb +3 -12
  16. data/app/models/rails_pulse/charts/percentile_chart_base.rb +1 -2
  17. data/app/models/rails_pulse/cleanup_run.rb +32 -0
  18. data/app/models/rails_pulse/cloud/buffered_batch.rb +55 -0
  19. data/app/models/rails_pulse/cloud/installation.rb +55 -0
  20. data/app/models/rails_pulse/dashboard/concerns/attention_classification.rb +104 -0
  21. data/app/models/rails_pulse/dashboard/concerns/time_range_helper.rb +24 -0
  22. data/app/models/rails_pulse/dashboard/health_summary.rb +9 -11
  23. data/app/models/rails_pulse/dashboard/needs_attention.rb +30 -96
  24. data/app/models/rails_pulse/dashboard/storage_pressure.rb +7 -61
  25. data/app/models/rails_pulse/dashboard/storage_status.rb +94 -39
  26. data/app/models/rails_pulse/event.rb +3 -2
  27. data/app/models/rails_pulse/exceptions/charts/occurrence_volume.rb +1 -3
  28. data/app/models/rails_pulse/jobs/charts/duration.rb +1 -2
  29. data/app/models/rails_pulse/jobs/charts/execution_volume.rb +1 -1
  30. data/app/models/rails_pulse/jobs/charts/failure_rate.rb +1 -2
  31. data/app/models/rails_pulse/like_pattern.rb +6 -0
  32. data/app/models/rails_pulse/queries/charts/database_load.rb +1 -3
  33. data/app/models/rails_pulse/queries/charts/execution_volume.rb +1 -1
  34. data/app/models/rails_pulse/routes/charts/error_rate.rb +1 -1
  35. data/app/models/rails_pulse/routes/charts/request_volume.rb +1 -1
  36. data/app/models/rails_pulse/time_window.rb +15 -0
  37. data/app/models/rails_pulse/writer_heartbeat.rb +6 -2
  38. data/app/serializers/rails_pulse/api/v1/deployment_serializer.rb +5 -2
  39. data/app/serializers/rails_pulse/api/v1/request_serializer.rb +1 -0
  40. data/app/services/rails_pulse/cloud/batch.rb +83 -0
  41. data/app/services/rails_pulse/cloud/client.rb +68 -0
  42. data/app/services/rails_pulse/cloud/combined_metrics.rb +49 -0
  43. data/app/services/rails_pulse/cloud/health_item.rb +77 -0
  44. data/app/services/rails_pulse/cloud/path_prefix.rb +50 -0
  45. data/app/services/rails_pulse/cloud/preview.rb +83 -0
  46. data/app/services/rails_pulse/cloud/query_shape.rb +195 -0
  47. data/app/services/rails_pulse/cloud/record_items.rb +60 -0
  48. data/app/services/rails_pulse/cloud/route_pattern.rb +111 -0
  49. data/app/services/rails_pulse/cloud/summary_items.rb +188 -0
  50. data/app/services/rails_pulse/cloud/sync.rb +275 -0
  51. data/app/services/rails_pulse/config_recommendations.rb +132 -0
  52. data/app/services/rails_pulse/deployment_comparison.rb +177 -0
  53. data/app/services/rails_pulse/operations/change_point.rb +1 -1
  54. data/app/services/rails_pulse/period_insights.rb +131 -0
  55. data/app/services/rails_pulse/sql_query_normalizer.rb +222 -122
  56. data/app/views/rails_pulse/dashboard/_storage_panel.html.erb +13 -3
  57. data/app/views/rails_pulse/storage/show.html.erb +2 -2
  58. data/config/routes.rb +4 -0
  59. data/db/rails_pulse_cloud_schema.rb +58 -0
  60. data/lib/generators/rails_pulse/base_methods.rb +18 -0
  61. data/lib/generators/rails_pulse/install_cloud_generator.rb +84 -0
  62. data/lib/generators/rails_pulse/templates/db/rails_pulse_cloud_schema.rb +58 -0
  63. data/lib/generators/rails_pulse/templates/migrations/install_rails_pulse_cloud_tables.rb +30 -0
  64. data/lib/generators/rails_pulse/templates/rails_pulse.rb +35 -6
  65. data/lib/generators/rails_pulse/upgrade_generator.rb +9 -16
  66. data/lib/rails_pulse/cleanup_service.rb +11 -0
  67. data/lib/rails_pulse/cli/agent_files/agents.md +13 -5
  68. data/lib/rails_pulse/cli/agent_files/claude_skill.md +66 -4
  69. data/lib/rails_pulse/cli/deployments.rb +16 -2
  70. data/lib/rails_pulse/cli/exceptions.rb +5 -0
  71. data/lib/rails_pulse/cli/insights.rb +70 -0
  72. data/lib/rails_pulse/cli/main.rb +4 -0
  73. data/lib/rails_pulse/cli/queries.rb +5 -0
  74. data/lib/rails_pulse/cli/requests.rb +9 -1
  75. data/lib/rails_pulse/cli/routes.rb +7 -0
  76. data/lib/rails_pulse/configuration.rb +77 -1
  77. data/lib/rails_pulse/installers/config_updater.rb +9 -2
  78. data/lib/rails_pulse/mcp/server.rb +3 -1
  79. data/lib/rails_pulse/mcp/tools/deployments.rb +47 -10
  80. data/lib/rails_pulse/mcp/tools/errors.rb +11 -2
  81. data/lib/rails_pulse/mcp/tools/exceptions.rb +16 -2
  82. data/lib/rails_pulse/mcp/tools/insights.rb +99 -0
  83. data/lib/rails_pulse/mcp/tools/queries.rb +7 -1
  84. data/lib/rails_pulse/mcp/tools/routes.rb +12 -1
  85. data/lib/rails_pulse/tasks/status_reporter.rb +175 -0
  86. data/lib/rails_pulse/tracker.rb +2 -1
  87. data/lib/rails_pulse/version.rb +1 -1
  88. data/lib/tasks/rails_pulse.rake +6 -0
  89. data/lib/tasks/rails_pulse_tasks.rake +7 -0
  90. metadata +30 -3
@@ -9,6 +9,30 @@ module RailsPulse
9
9
 
10
10
  [ @period.days.ago.beginning_of_day, Time.current ]
11
11
  end
12
+
13
+ # Summary rows that cover the period exactly once. The same traffic is
14
+ # summarized per hour, day, week and month, so summing across period
15
+ # types would count it once per granularity. A window of 25 hours or
16
+ # less reads hourly rows. A longer one reads daily rows up to the start
17
+ # of today, or to its own end if that comes first, and hourly rows
18
+ # after today's start, because a day is summarized only once it has
19
+ # ended.
20
+ def period_summaries
21
+ start, finish = period_range
22
+ return RailsPulse::Summary.where(period_type: "hour", period_start: start..finish) if hourly_period?(start, finish)
23
+
24
+ today = Time.zone.now.beginning_of_day
25
+ RailsPulse::Summary.where(period_type: "day", period_start: start...[ today, finish ].min)
26
+ .or(RailsPulse::Summary.where(period_type: "hour", period_start: today..finish))
27
+ end
28
+
29
+ # The dashboard passes the period type TimeRange resolved, so these
30
+ # panels read the same granularity as the cards and charts beside them.
31
+ def hourly_period?(start, finish)
32
+ return @period_type == "hour" if @period_type
33
+
34
+ finish - start <= 25.hours
35
+ end
12
36
  end
13
37
  end
14
38
  end
@@ -4,11 +4,13 @@ module RailsPulse
4
4
  include Concerns::ThresholdConstants
5
5
  include Concerns::TimeRangeHelper
6
6
 
7
- def initialize(disabled_tags: [], show_non_tagged: true, period: 7, window: nil)
7
+ def initialize(disabled_tags: [], show_non_tagged: true, period: 7, period_type: nil, window: nil, storage_pressure: nil)
8
8
  @disabled_tags = disabled_tags
9
9
  @show_non_tagged = show_non_tagged
10
10
  @period = period
11
+ @period_type = period_type
11
12
  @window = window
13
+ @storage_pressure = storage_pressure
12
14
  @route_thresholds = RailsPulse.configuration.route_thresholds
13
15
  @query_thresholds = RailsPulse.configuration.query_thresholds
14
16
  @job_thresholds = RailsPulse.configuration.job_thresholds
@@ -29,11 +31,10 @@ module RailsPulse
29
31
  private
30
32
 
31
33
  def route_counts
32
- start, finish = period_range
33
-
34
34
  data = RailsPulse::Summary
35
35
  .with_tag_filters(@disabled_tags, @show_non_tagged)
36
- .where(summarizable_type: "RailsPulse::Route", period_start: start..finish)
36
+ .where(summarizable_type: "RailsPulse::Route")
37
+ .merge(period_summaries)
37
38
  .group("summarizable_id")
38
39
  .select(
39
40
  "summarizable_id",
@@ -62,11 +63,10 @@ module RailsPulse
62
63
  end
63
64
 
64
65
  def query_counts
65
- start, finish = period_range
66
-
67
66
  data = RailsPulse::Summary
68
67
  .with_tag_filters(@disabled_tags, @show_non_tagged)
69
- .where(summarizable_type: "RailsPulse::Query", period_start: start..finish)
68
+ .where(summarizable_type: "RailsPulse::Query")
69
+ .merge(period_summaries)
70
70
  .group("summarizable_id")
71
71
  .select(
72
72
  "summarizable_id",
@@ -90,7 +90,7 @@ module RailsPulse
90
90
  end
91
91
 
92
92
  def storage_counts
93
- StoragePressure.new.storage_counts
93
+ (@storage_pressure || StoragePressure.new).storage_counts
94
94
  end
95
95
 
96
96
  # A process that dropped and has since gone away still counts as
@@ -152,12 +152,10 @@ module RailsPulse
152
152
  return nil unless RailsPulse.configuration.track_exceptions
153
153
  return nil unless exception_summaries_available?
154
154
 
155
- start, finish = period_range
156
-
157
155
  counts = RailsPulse::Summary
158
156
  .for_exceptions
159
157
  .where.not(summarizable_id: 0)
160
- .where(period_start: start..finish)
158
+ .merge(period_summaries)
161
159
  .group(:summarizable_id)
162
160
  .sum(:count)
163
161
 
@@ -1,14 +1,16 @@
1
1
  module RailsPulse
2
2
  module Dashboard
3
3
  class NeedsAttention
4
- include Concerns::ThresholdConstants
4
+ include Concerns::AttentionClassification
5
5
  include Concerns::TimeRangeHelper
6
6
 
7
- def initialize(disabled_tags: [], show_non_tagged: true, period: 7, window: nil)
7
+ def initialize(disabled_tags: [], show_non_tagged: true, period: 7, period_type: nil, window: nil, storage_pressure: nil)
8
8
  @disabled_tags = disabled_tags
9
9
  @show_non_tagged = show_non_tagged
10
10
  @period = period
11
+ @period_type = period_type
11
12
  @window = window
13
+ @storage_pressure = storage_pressure
12
14
  @route_thresholds = RailsPulse.configuration.route_thresholds
13
15
  @query_thresholds = RailsPulse.configuration.query_thresholds
14
16
  @job_thresholds = RailsPulse.configuration.job_thresholds
@@ -40,7 +42,18 @@ module RailsPulse
40
42
  private
41
43
 
42
44
  def storage_pressure_items
43
- StoragePressure.new.pressure_items
45
+ (@storage_pressure || StoragePressure.new).pressure_items
46
+ end
47
+
48
+ # The dashboard window is user-selected, so the error-rate reason names
49
+ # the window's real length rather than assuming a week.
50
+ def span_phrase
51
+ start, finish = period_range
52
+ hours = ((finish - start) / 1.hour).round
53
+ return "in the last #{hours} hour#{"s" unless hours == 1}" if hours <= 25
54
+
55
+ days = (hours / 24.0).round
56
+ "in the last #{days} day#{"s" unless days == 1}"
44
57
  end
45
58
 
46
59
  def url_helpers
@@ -48,12 +61,11 @@ module RailsPulse
48
61
  end
49
62
 
50
63
  def route_items
51
- start, finish = period_range
52
-
53
64
  route_data = RailsPulse::Summary
54
65
  .with_tag_filters(@disabled_tags, @show_non_tagged)
55
66
  .joins("INNER JOIN rails_pulse_routes ON rails_pulse_routes.id = rails_pulse_summaries.summarizable_id")
56
- .where(summarizable_type: "RailsPulse::Route", period_start: start..finish)
67
+ .where(summarizable_type: "RailsPulse::Route")
68
+ .merge(period_summaries)
57
69
  .group("rails_pulse_summaries.summarizable_id, rails_pulse_routes.path")
58
70
  .select(
59
71
  "rails_pulse_summaries.summarizable_id as route_id",
@@ -70,7 +82,7 @@ module RailsPulse
70
82
  errors = record.total_errors.to_i
71
83
  error_rate = total > 0 ? (errors * 100.0 / total).round(1) : 0.0
72
84
 
73
- severity, reason, metric, metric_sub, sort_score = classify_route(p95, total, errors, error_rate)
85
+ severity, reason, metric, metric_sub, sort_score = classify_route(p95, total, errors, error_rate, span_phrase)
74
86
  next unless severity
75
87
 
76
88
  items << {
@@ -88,45 +100,12 @@ module RailsPulse
88
100
  items
89
101
  end
90
102
 
91
- def classify_route(p95, total, errors, error_rate)
92
- if p95 >= @route_thresholds[:critical] || error_rate >= CRITICAL_ERROR_RATE
93
- if error_rate >= CRITICAL_ERROR_RATE
94
- [ :critical,
95
- "#{error_rate}% error rate · #{total} requests this week",
96
- "#{errors} errors",
97
- "P95 #{p95.round(0).to_i}ms",
98
- errors * total.to_f ]
99
- else
100
- [ :critical,
101
- "#{p95.round(0).to_i}ms P95 · exceeds #{@route_thresholds[:critical]}ms threshold",
102
- "#{p95.round(0).to_i}ms P95",
103
- "#{total} requests",
104
- p95 ]
105
- end
106
- elsif p95 >= @route_thresholds[:slow] || error_rate >= WARNING_ERROR_RATE
107
- if error_rate >= WARNING_ERROR_RATE && p95 < @route_thresholds[:slow]
108
- [ :warning,
109
- "#{error_rate}% error rate · #{total} requests this week",
110
- "#{errors} errors",
111
- "P95 #{p95.round(0).to_i}ms",
112
- errors * total.to_f ]
113
- else
114
- [ :warning,
115
- "#{p95.round(0).to_i}ms P95 · #{error_rate > 0 ? "#{error_rate}% error rate" : "above slow threshold"}",
116
- "#{p95.round(0).to_i}ms P95",
117
- "#{total} requests",
118
- p95 ]
119
- end
120
- end
121
- end
122
-
123
103
  def query_items
124
- start, finish = period_range
125
-
126
104
  query_data = RailsPulse::Summary
127
105
  .with_tag_filters(@disabled_tags, @show_non_tagged)
128
106
  .joins("INNER JOIN rails_pulse_queries ON rails_pulse_queries.id = rails_pulse_summaries.summarizable_id")
129
- .where(summarizable_type: "RailsPulse::Query", period_start: start..finish)
107
+ .where(summarizable_type: "RailsPulse::Query")
108
+ .merge(period_summaries)
130
109
  .group("rails_pulse_summaries.summarizable_id, rails_pulse_queries.normalized_sql")
131
110
  .select(
132
111
  "rails_pulse_summaries.summarizable_id as query_id",
@@ -142,26 +121,19 @@ module RailsPulse
142
121
  p95 = record.p95_duration.to_f
143
122
  count = record.total_count.to_i
144
123
 
145
- if p95 >= @query_thresholds[:critical]
146
- severity = :critical
147
- reason = "#{p95.round(0).to_i}ms P95 · exceeds #{@query_thresholds[:critical]}ms threshold"
148
- elsif p95 >= @query_thresholds[:slow]
149
- severity = :warning
150
- reason = "#{p95.round(0).to_i}ms P95 · above #{@query_thresholds[:slow]}ms slow threshold"
151
- else
152
- next
153
- end
124
+ severity, reason, metric, metric_sub, sort_score = classify_query(p95, count)
125
+ next unless severity
154
126
 
155
127
  classified_ids << record.query_id
156
128
  items << {
157
129
  type: "QUERY",
158
130
  name: truncate_sql(record.normalized_sql),
159
131
  reason: reason,
160
- metric: "#{p95.round(0).to_i}ms P95",
161
- metric_sub: "#{count} execution#{count == 1 ? "" : "s"}",
132
+ metric: metric,
133
+ metric_sub: metric_sub,
162
134
  link: url_helpers.query_path(record.query_id),
163
135
  severity: severity,
164
- sort_score: p95,
136
+ sort_score: sort_score,
165
137
  monospace: true
166
138
  }
167
139
  end
@@ -199,7 +171,9 @@ module RailsPulse
199
171
  failure_rate = job.failure_rate
200
172
  p95 = job.p95_duration.to_f
201
173
 
202
- severity, reason, metric, metric_sub, sort_score = classify_job(job, failure_rate, p95)
174
+ severity, reason, metric, metric_sub, sort_score = classify_job(
175
+ failure_rate, p95, job.runs_count, job.failures_count, job.queue_name.presence || "default"
176
+ )
203
177
  next unless severity
204
178
 
205
179
  items << {
@@ -217,38 +191,6 @@ module RailsPulse
217
191
  items
218
192
  end
219
193
 
220
- def classify_job(job, failure_rate, p95)
221
- if failure_rate >= CRITICAL_JOB_FAILURE_RATE || p95 >= @job_thresholds[:critical]
222
- if failure_rate >= CRITICAL_JOB_FAILURE_RATE
223
- [ :critical,
224
- "#{job.queue_name.presence || "default"} queue · #{failure_rate}% failure rate",
225
- "#{job.failures_count} / #{job.runs_count} failed",
226
- "P95 #{p95.round(0).to_i}ms",
227
- job.failures_count * job.runs_count.to_f ]
228
- else
229
- [ :critical,
230
- "P95 #{p95.round(0).to_i}ms · exceeds #{@job_thresholds[:critical]}ms threshold",
231
- "#{p95.round(0).to_i}ms P95",
232
- "#{job.runs_count} runs",
233
- p95 ]
234
- end
235
- elsif failure_rate >= WARNING_JOB_FAILURE_RATE || p95 >= @job_thresholds[:slow]
236
- if failure_rate >= WARNING_JOB_FAILURE_RATE
237
- [ :warning,
238
- "#{job.queue_name.presence || "default"} queue · #{failure_rate}% failure rate",
239
- "#{job.failures_count} / #{job.runs_count} failed",
240
- "P95 #{p95.round(0).to_i}ms",
241
- job.failures_count * job.runs_count.to_f ]
242
- else
243
- [ :warning,
244
- "P95 #{p95.round(0).to_i}ms · above slow threshold",
245
- "#{p95.round(0).to_i}ms P95",
246
- "#{job.runs_count} runs",
247
- p95 ]
248
- end
249
- end
250
- end
251
-
252
194
  # Exception groups that fired often enough over the period to be worth
253
195
  # looking at. Read from summaries, not from ExceptionGroup#occurrence_count
254
196
  # — that is a lifetime counter, so a group that was noisy last year and is
@@ -261,12 +203,10 @@ module RailsPulse
261
203
  return [] unless RailsPulse.configuration.track_exceptions
262
204
  return [] unless exception_tables_available?
263
205
 
264
- start, finish = period_range
265
-
266
206
  counts = RailsPulse::Summary
267
207
  .for_exceptions
268
208
  .where.not(summarizable_id: 0)
269
- .where(period_start: start..finish)
209
+ .merge(period_summaries)
270
210
  .group(:summarizable_id)
271
211
  .sum(:count)
272
212
 
@@ -319,12 +259,6 @@ module RailsPulse
319
259
  rescue ActiveRecord::ActiveRecordError
320
260
  false
321
261
  end
322
-
323
- def truncate_sql(sql)
324
- return "" if sql.blank?
325
- cleaned = sql.gsub(/\s+/, " ").strip
326
- cleaned.length > 80 ? "#{cleaned[0..79]}..." : cleaned
327
- end
328
262
  end
329
263
  end
330
264
  end
@@ -8,8 +8,10 @@ module RailsPulse
8
8
  @config = RailsPulse.configuration
9
9
  end
10
10
 
11
+ # Memoized: computing the items costs several aggregate queries (one a
12
+ # raw-request count), and one dashboard render asks for them repeatedly.
11
13
  def pressure_items
12
- summary_staleness_items + stuck_records_items + sub_hour_retention_items + writer_drop_items
14
+ @pressure_items ||= summary_staleness_items + sub_hour_retention_items + writer_drop_items
13
15
  end
14
16
 
15
17
  # Dropped requests get their own Tracking badge, not this one
@@ -47,43 +49,7 @@ module RailsPulse
47
49
  end
48
50
  end
49
51
 
50
- # Signal B — Records past retention that haven't been summarized (cleanup is blocked)
51
- def stuck_records_items
52
- return [] unless @config.archiving_enabled
53
- return [] unless @config.full_retention_period
54
-
55
- oldest_start = oldest_overall_request_summary_start
56
- return [] if oldest_start.nil?
57
-
58
- stuck_count = RailsPulse::Request
59
- .where("occurred_at < ?", @config.full_retention_period.ago)
60
- .where("occurred_at < ?", oldest_start)
61
- .count
62
-
63
- return [] if stuck_count.zero?
64
-
65
- [ {
66
- type: "STORAGE",
67
- name: "Storage pressure",
68
- reason: "#{stuck_count} #{"request".pluralize(stuck_count)} past the retention window cannot be cleaned up — summarize first",
69
- metric: "#{stuck_count} stuck",
70
- metric_sub: "older than #{humanize_duration(@config.full_retention_period)}",
71
- link: "#",
72
- severity: :critical,
73
- sort_score: stuck_count.to_f,
74
- popover_title: "Cleanup is blocked on unsummarised records",
75
- popover_body: "#{stuck_count} #{"request".pluralize(stuck_count)} are older than your retention period " \
76
- "(#{humanize_duration(@config.full_retention_period)}) but predate any hourly summary record. " \
77
- "CleanupService will not delete records from a time period until that period has been summarised — " \
78
- "otherwise those data points would be permanently lost from dashboard charts.<br><br>" \
79
- "<strong>How to fix:</strong> Run <code>RailsPulse::SummaryJob</code> to summarise the historical periods. " \
80
- "If the job is not yet scheduled, run the backfill task to summarise all historical data:<br><br>" \
81
- "<code>rails rails_pulse:backfill_summaries</code><br><br>" \
82
- "Once the affected periods are summarised, the next cleanup run will delete these records automatically."
83
- } ]
84
- end
85
-
86
- # Signal C — Retention period shorter than 1-hour cleanup minimum
52
+ # Signal B — Retention period shorter than 1-hour cleanup minimum
87
53
  def sub_hour_retention_items
88
54
  return [] unless @config.full_retention_period
89
55
  return [] unless @config.full_retention_period < 1.hour
@@ -108,7 +74,7 @@ module RailsPulse
108
74
  } ]
109
75
  end
110
76
 
111
- # Signal D — the background writer is discarding requests
77
+ # Signal C — the background writer is discarding requests
112
78
  def writer_drop_items
113
79
  return [] unless RailsPulse::Event.table_available?
114
80
 
@@ -148,8 +114,8 @@ module RailsPulse
148
114
  severity: severity,
149
115
  sort_score: severity == :critical ? Float::INFINITY : 1.0,
150
116
  popover_title: "Summary job is not running",
151
- popover_body: "Rails Pulse summarises raw request data into hourly aggregates before cleanup can safely delete them. " \
152
- "Until a period is summarised, CleanupService will not delete records from it — so your database will grow until the summary job catches up.<br><br>" \
117
+ popover_body: "Rails Pulse summarises raw request data into hourly aggregates for the dashboard's charts. " \
118
+ "Count-based cleanup only trims records from periods that have been summarised, so until the job catches up tables can grow past <code>max_table_records</code>.<br><br>" \
153
119
  "<strong>How to fix:</strong> Schedule <code>RailsPulse::SummaryJob</code> to run every hour in your job scheduler " \
154
120
  "(Sidekiq-Cron, GoodJob, Solid Queue, etc.). Once it has run, cleanup will resume automatically on its next execution.<br><br>" \
155
121
  "To backfill any historical gaps, run: <code>rails rails_pulse:backfill_summaries</code>"
@@ -161,26 +127,6 @@ module RailsPulse
161
127
  .where(summarizable_type: "RailsPulse::Request", summarizable_id: 0, period_type: "hour")
162
128
  .maximum(:period_end)
163
129
  end
164
-
165
- def oldest_overall_request_summary_start
166
- RailsPulse::Summary
167
- .where(summarizable_type: "RailsPulse::Request", summarizable_id: 0, period_type: "hour")
168
- .minimum(:period_start)
169
- end
170
-
171
- def humanize_duration(duration)
172
- total_seconds = duration.to_i
173
- if total_seconds >= 86_400
174
- days = total_seconds / 86_400
175
- "#{days} #{"day".pluralize(days)}"
176
- elsif total_seconds >= 3_600
177
- hours = total_seconds / 3_600
178
- "#{hours} #{"hour".pluralize(hours)}"
179
- else
180
- minutes = total_seconds / 60
181
- "#{minutes} #{"minute".pluralize(minutes)}"
182
- end
183
- end
184
130
  end
185
131
  end
186
132
  end
@@ -34,7 +34,7 @@ module RailsPulse
34
34
  name: :rails_pulse_summaries,
35
35
  model: "RailsPulse::Summary",
36
36
  label: "Summaries",
37
- description: "Hourly and daily aggregates used by charts. Hourly rows older than 2 days are cleaned up",
37
+ description: "Hourly and daily aggregates used by charts. Hourly rows are cleaned up after 7 days by default (hourly_summary_retention)",
38
38
  time_column: :period_start
39
39
  },
40
40
  {
@@ -78,42 +78,43 @@ module RailsPulse
78
78
  label: "Events",
79
79
  description: "What Pulse noticed: writer heartbeats every minute, pruned after a day; any other kind is kept for event_retention_period",
80
80
  time_column: :occurred_at
81
+ },
82
+ {
83
+ name: :rails_pulse_cloud_batches,
84
+ model: "RailsPulse::Cloud::BufferedBatch",
85
+ label: "Cloud buffer",
86
+ description: "Batches waiting to be sent to Rails Pulse Cloud. Held for at most 7 days or 50 MB",
87
+ time_column: :created_at
81
88
  }
82
89
  ].freeze
83
90
 
84
- # Table sizes come from dbstat / pg_total_relation_size /
85
- # information_schema, one statement per table. The dashboard only shows
86
- # a headline total, so it reads them through a short-lived per-process
87
- # cache; the Storage page itself always measures afresh.
88
- SIZE_CACHE_TTL = 5.minutes
89
- @size_cache = {}
90
- @size_cache_mutex = Mutex.new
91
-
92
- class << self
93
- def cached_table_bytes(table_name)
94
- @size_cache_mutex.synchronize do
95
- entry = @size_cache[table_name]
96
- return entry[:bytes] if entry && entry[:measured_at] > SIZE_CACHE_TTL.ago
97
-
98
- bytes = yield
99
- @size_cache[table_name] = { bytes: bytes, measured_at: Time.current }
100
- bytes
101
- end
102
- end
91
+ # The Storage page counts every row exactly: one unindexed COUNT/MIN/MAX
92
+ # scan per table, which takes tens of seconds on large installs. The
93
+ # dashboard only shows headline counts and fill bars, so with
94
+ # estimated: true it reads the catalog's row estimates instead
95
+ # (pg_class.reltuples, information_schema TABLE_ROWS). SQLite keeps no
96
+ # such estimate, and dbstat walks the whole file, so there the dashboard
97
+ # measures no tables at all.
98
+ def initialize(estimated: false, storage_pressure: nil)
99
+ @config = RailsPulse.configuration
100
+ @pressure = storage_pressure || StoragePressure.new
101
+ @estimated = estimated
102
+ end
103
103
 
104
- def reset_size_cache!
105
- @size_cache_mutex.synchronize { @size_cache.clear }
106
- end
104
+ def estimated?
105
+ @estimated
107
106
  end
108
107
 
109
- def initialize(cached_sizes: false)
110
- @config = RailsPulse.configuration
111
- @pressure = StoragePressure.new
112
- @cached_sizes = cached_sizes
108
+ def counts_available?
109
+ !(@estimated && sqlite?)
113
110
  end
114
111
 
115
112
  def tables
116
- @tables ||= TABLE_CATALOG.filter_map { |definition| build_table(definition) }
113
+ @tables ||= if counts_available?
114
+ with_current_mysql_statistics { TABLE_CATALOG.filter_map { |definition| build_table(definition) } }
115
+ else
116
+ []
117
+ end
117
118
  end
118
119
 
119
120
  def pressure_items
@@ -198,12 +199,37 @@ module RailsPulse
198
199
  RailsPulse::ApplicationRecord.connection
199
200
  end
200
201
 
202
+ # MySQL 8 caches information_schema's TABLE_ROWS, DATA_LENGTH and
203
+ # INDEX_LENGTH for information_schema_stats_expiry seconds, a day by
204
+ # default, so a table that grew after its first read reports its old
205
+ # size. An expiry of 0 reads current statistics; a SET_VAR hint does
206
+ # not reach information_schema, so it is set on the session and put
207
+ # back afterwards because the connection returns to the pool. MariaDB
208
+ # has no such variable and does not cache, so the block runs as is.
209
+ def with_current_mysql_statistics
210
+ previous = session_statistics_expiry
211
+ return yield if previous.nil?
212
+
213
+ connection.execute("SET SESSION information_schema_stats_expiry = 0")
214
+ yield
215
+ ensure
216
+ connection.execute("SET SESSION information_schema_stats_expiry = #{previous.to_i}") unless previous.nil?
217
+ end
218
+
219
+ def session_statistics_expiry
220
+ return unless mysql?
221
+
222
+ connection.select_value("SELECT @@SESSION.information_schema_stats_expiry")
223
+ rescue ActiveRecord::StatementInvalid
224
+ nil
225
+ end
226
+
201
227
  def build_table(definition)
202
228
  model = definition[:model].safe_constantize
203
229
  return unless model
204
230
  return unless connection.table_exists?(definition[:name])
205
231
 
206
- stats = fetch_counts(model, definition)
232
+ stats = measured_counts(model, definition)
207
233
  limit = @config.max_table_records&.[](definition[:name])
208
234
  count = stats[:count]
209
235
  percent = limit.to_i.positive? ? ((count.to_f / limit) * 100).round(1) : nil
@@ -223,7 +249,7 @@ module RailsPulse
223
249
  newest_at: stats[:newest_at],
224
250
  recent_count: stats[:recent_count],
225
251
  runway_label: runway_label(count, limit, stats[:recent_count]),
226
- bytes: measured_table_bytes(definition[:name]),
252
+ bytes: table_bytes(definition[:name]),
227
253
  history_label: history_label(stats[:oldest_at], stats[:newest_at])
228
254
  }
229
255
  rescue => error
@@ -247,6 +273,33 @@ module RailsPulse
247
273
  }
248
274
  end
249
275
 
276
+ def measured_counts(model, definition)
277
+ return fetch_counts(model, definition) unless @estimated
278
+
279
+ { count: estimated_count(definition[:name]), oldest_at: nil, newest_at: nil, recent_count: 0 }
280
+ end
281
+
282
+ def estimated_count(table_name)
283
+ if connection.adapter_name.downcase.include?("postgres")
284
+ estimate = connection.select_value(
285
+ "SELECT reltuples::bigint FROM pg_class WHERE oid = to_regclass(#{connection.quote(table_name.to_s)})"
286
+ ).to_i
287
+ # -1 until the table's first ANALYZE; autovacuum runs one after
288
+ # about 50 inserts, so a table without an estimate is small enough
289
+ # to count.
290
+ estimate.negative? ? connection.select_value("SELECT COUNT(*) FROM #{connection.quote_table_name(table_name)}").to_i : estimate
291
+ else
292
+ connection.select_value(
293
+ RailsPulse::ApplicationRecord.sanitize_sql_array([ <<~SQL, table_name.to_s ])
294
+ SELECT TABLE_ROWS
295
+ FROM information_schema.TABLES
296
+ WHERE TABLE_SCHEMA = DATABASE()
297
+ AND TABLE_NAME = ?
298
+ SQL
299
+ ).to_i
300
+ end
301
+ end
302
+
250
303
  def fetch_counts(model, definition)
251
304
  quoted_table = connection.quote_table_name(definition[:name])
252
305
  time_column = definition[:time_column]
@@ -351,12 +404,6 @@ module RailsPulse
351
404
  row[key] || row[key.to_s] || row[key.to_s.upcase] || row[key.to_sym]
352
405
  end
353
406
 
354
- def measured_table_bytes(table_name)
355
- return table_bytes(table_name) unless @cached_sizes
356
-
357
- self.class.cached_table_bytes(table_name) { table_bytes(table_name) }
358
- end
359
-
360
407
  def table_bytes(table_name)
361
408
  adapter = connection.adapter_name.downcase
362
409
 
@@ -364,7 +411,7 @@ module RailsPulse
364
411
  connection.select_value(
365
412
  "SELECT pg_total_relation_size(#{connection.quote(table_name.to_s)})"
366
413
  ).to_i
367
- elsif adapter.include?("mysql")
414
+ elsif mysql?
368
415
  connection.select_value(
369
416
  RailsPulse::ApplicationRecord.sanitize_sql_array([ <<~SQL, table_name.to_s ])
370
417
  SELECT DATA_LENGTH + INDEX_LENGTH
@@ -410,11 +457,19 @@ module RailsPulse
410
457
  nil
411
458
  end
412
459
 
460
+ def mysql?
461
+ connection.adapter_name.downcase.match?(/mysql|trilogy/)
462
+ end
463
+
464
+ def sqlite?
465
+ connection.adapter_name.downcase.include?("sqlite")
466
+ end
467
+
413
468
  def adapter_label
414
469
  name = connection.adapter_name
415
470
  case name.downcase
416
471
  when /postgres/ then "PostgreSQL"
417
- when /mysql/ then "MySQL"
472
+ when /mysql|trilogy/ then "MySQL"
418
473
  when /sqlite/ then "SQLite"
419
474
  else name
420
475
  end
@@ -443,7 +498,7 @@ module RailsPulse
443
498
  "File size of the dedicated Rails Pulse SQLite database."
444
499
  elsif adapter.include?("postgres")
445
500
  "PostgreSQL relation size including indexes (pg_total_relation_size)."
446
- elsif adapter.include?("mysql")
501
+ elsif mysql?
447
502
  "MySQL data + index length from information_schema."
448
503
  end
449
504
  end
@@ -3,8 +3,9 @@ module RailsPulse
3
3
  # outcome or sample, tagged by `kind`; `subject` names what it is about,
4
4
  # `value` carries its number and `metadata` the kind-specific detail as JSON.
5
5
  #
6
- # The one kind written today is writer_heartbeat (see WriterHeartbeat). The
7
- # table is generic so a new kind needs no migration. Decision 0019.
6
+ # Two kinds are written today: writer_heartbeat (see WriterHeartbeat) and
7
+ # cleanup_run (see CleanupRun). The table is generic so a new kind needs no
8
+ # migration. Decision 0019.
8
9
  #
9
10
  # CleanupService deletes rows older than config.event_retention_period,
10
11
  # except kinds listed in config.event_retention_exempt_kinds (rows a writer
@@ -41,10 +41,8 @@ module RailsPulse
41
41
  # answer "zero" rather than "no data" — leaving the gap unfilled would
42
42
  # draw a chart that skips quiet days and misrepresents the trend.
43
43
  def pad_with_zeros(raw)
44
- step = @period_type.to_s == "hour" ? 3600 : 86_400
45
-
46
44
  {}.tap do |padded|
47
- (@window.start_time.to_i..@window.end_time.to_i).step(step) do |timestamp|
45
+ @window.bucket_timestamps(@period_type).each do |timestamp|
48
46
  padded[timestamp] = raw[timestamp] || 0
49
47
  end
50
48
  end
@@ -31,9 +31,8 @@ module RailsPulse
31
31
  end
32
32
 
33
33
  # Convert to final values (weighted averages) and pad missing data
34
- step = time_step
35
34
  daily_data = {}
36
- (@window.start_time.to_i..@window.end_time.to_i).step(step) do |timestamp|
35
+ @window.bucket_timestamps(@period_type).each do |timestamp|
37
36
  if raw_data[timestamp]
38
37
  count = raw_data[timestamp][:total_count]
39
38
  daily_data[timestamp] = {