rails_pulse 0.5.0 → 0.5.1.pre.1

data/app/services/rails_pulse/summary_service/from_raw_rows.rb

@@ -10,6 +10,13 @@ module RailsPulse
     # hour costs one query per kind however many routes, queries or jobs it
     # saw. Percentiles are exact.
     #
+    # When detail capture folded part of the hour into rails_pulse_rollups
+    # (decision 0022), each subject's raw rows are merged with its rollups:
+    # counts, sums, min, max, averages and stddev stay exact, and percentiles
+    # come from the rollups' histograms walked together with the exactly-kept
+    # durations. A subject with no rollups takes the plain path, so with
+    # detail capture off nothing changes.
+    #
     # Rows carry the summarizable and its metrics; SummaryService adds the
     # period columns.
     class FromRawRows
@@ -21,29 +28,38 @@ module RailsPulse
       def request_rows
         durations = request_data.map { |_, duration, _| duration }.compact.sort
         statuses = request_data.map { |_, _, status| status }
+        rollup = merge_rollups(route_rollups.values)
 
-        [ row("RailsPulse::Request", 0).merge(Metrics.duration(durations), Metrics.status(statuses)) ]
+        [ row("RailsPulse::Request", 0).merge(duration_metrics(durations, rollup), status_metrics(statuses, rollup)) ]
       end
 
       def route_rows
-        request_data.slice_when { |previous, current| previous.first != current.first }.map do |rows|
+        raw_by_route = request_data
+          .slice_when { |previous, current| previous.first != current.first }
+          .to_h { |rows| [ rows.first.first, rows ] }
+
+        (raw_by_route.keys | route_rollups.keys).map do |route_id|
+          rows = raw_by_route[route_id] || []
           durations = rows.map { |_, duration, _| duration }
           statuses = rows.map { |_, _, status| status }
+          rollup = route_rollups[route_id]
 
-          row("RailsPulse::Route", rows.first.first).merge(
-            Metrics.duration(durations),
-            Metrics.status(statuses),
-            count: rows.size
+          row("RailsPulse::Route", route_id).merge(
+            duration_metrics(durations, rollup),
+            status_metrics(statuses, rollup)
           )
         end
       end
 
       def query_rows
-        rows_by_summarizable(Operation.where(occurred_at: @time_range), :query_id).filter_map do |query_id, rows|
-          durations = rows.map(&:first).compact
-          next if durations.empty?
+        raw_by_query = rows_by_summarizable(Operation.where(occurred_at: @time_range), :query_id).to_h
 
-          row("RailsPulse::Query", query_id).merge(Metrics.duration(durations))
+        (raw_by_query.keys | query_rollups.keys).filter_map do |query_id|
+          durations = (raw_by_query[query_id] || []).map(&:first).compact
+          rollup = query_rollups[query_id]
+          next if durations.empty? && rollup.nil?
+
+          row("RailsPulse::Query", query_id).merge(duration_metrics(durations, rollup))
         end
       end
 
@@ -84,10 +100,144 @@ module RailsPulse
 
       private
 
+      # Everything one subject's rollup rows add up to, with sums as floats.
+      MergedRollup = Struct.new(
+        :count, :error_count,
+        :status_2xx, :status_3xx, :status_4xx, :status_5xx,
+        :sum_duration, :sum_square_duration, :min_duration, :max_duration,
+        :histogram_counts
+      )
+
       def row(summarizable_type, summarizable_id)
         { summarizable_type: summarizable_type, summarizable_id: summarizable_id }
       end
 
+      # Identical to Metrics.duration when the subject has no rollups.
+      def duration_metrics(sorted_durations, rollup)
+        return Metrics.duration(sorted_durations) if rollup.nil?
+
+        count = sorted_durations.size + rollup.count
+        total = sorted_durations.sum + rollup.sum_duration
+        sum_of_squares = sorted_durations.sum { |d| d * d } + rollup.sum_square_duration
+        avg = count.positive? ? total.to_f / count : 0
+
+        {
+          count: count,
+          avg_duration: avg,
+          min_duration: [ sorted_durations.first, rollup.min_duration ].compact.min,
+          max_duration: [ sorted_durations.last, rollup.max_duration ].compact.max,
+          total_duration: total,
+          p50_duration: merged_percentile(rollup, sorted_durations, 0.5),
+          p95_duration: merged_percentile(rollup, sorted_durations, 0.95),
+          p99_duration: merged_percentile(rollup, sorted_durations, 0.99),
+          # Each value as its own group makes this the exact sample stddev
+          # from N, sum(x) and sum(x^2).
+          stddev_duration: Statistics.pooled_stddev(
+            count: count, within_sum: 0,
+            weighted_mean_sum: total, weighted_square_sum: sum_of_squares
+          )
+        }
+      end
+
+      def status_metrics(statuses, rollup)
+        return Metrics.status(statuses) if rollup.nil?
+
+        base = Metrics.status(statuses)
+        {
+          error_count: base[:error_count] + rollup.error_count,
+          success_count: base[:success_count] + (rollup.count - rollup.error_count),
+          status_2xx: base[:status_2xx] + rollup.status_2xx,
+          status_3xx: base[:status_3xx] + rollup.status_3xx,
+          status_4xx: base[:status_4xx] + rollup.status_4xx,
+          status_5xx: base[:status_5xx] + rollup.status_5xx
+        }
+      end
+
+      def merged_percentile(rollup, sorted_durations, percentile)
+        DurationHistogram.percentile(
+          counts: rollup.histogram_counts,
+          exact_sorted: sorted_durations,
+          percentile: percentile
+        )
+      end
+
+      def route_rollups
+        rollup_data.fetch("RailsPulse::Route", {})
+      end
+
+      def query_rollups
+        rollup_data.fetch("RailsPulse::Query", {})
+      end
+
+      # { rollupable_type => { rollupable_id => MergedRollup } } for the hour,
+      # each subject's writer rows already summed. Empty on installs that have
+      # not migrated the table yet.
+      def rollup_data
+        @rollup_data ||= Rollup.table_available? ? load_rollups : {}
+      end
+
+      def load_rollups
+        Rollup
+          .where(period_start: @time_range)
+          .pluck(:rollupable_type, :rollupable_id, :count, :error_count,
+                 :status_2xx, :status_3xx, :status_4xx, :status_5xx,
+                 :sum_duration, :sum_square_duration, :min_duration, :max_duration,
+                 :histogram)
+          .group_by { |type, _id, *| type }
+          .transform_values do |type_rows|
+            type_rows
+              .group_by { |_type, id, *| id }
+              .transform_values { |subject_rows| merge_rollup_rows(subject_rows) }
+          end
+      end
+
+      def merge_rollup_rows(rows)
+        merged = MergedRollup.new(0, 0, 0, 0, 0, 0, 0.0, 0.0, nil, nil, {})
+        rows.each do |_type, _id, count, error_count, s2, s3, s4, s5, sum, sum_sq, min, max, histogram|
+          merged.count += count
+          merged.error_count += error_count
+          merged.status_2xx += s2
+          merged.status_3xx += s3
+          merged.status_4xx += s4
+          merged.status_5xx += s5
+          merged.sum_duration += sum.to_f
+          merged.sum_square_duration += sum_sq.to_f
+          merged.min_duration = [ merged.min_duration, min&.to_f ].compact.min
+          merged.max_duration = [ merged.max_duration, max&.to_f ].compact.max
+          add_histogram(merged.histogram_counts, histogram)
+        end
+        merged
+      end
+
+      def add_histogram(accumulated, histogram_json)
+        DurationHistogram.load(histogram_json).counts.each do |index, n|
+          accumulated[index] = accumulated.fetch(index, 0) + n
+        end
+      end
+
+      # The overall row folds every route's rollups into one.
+      def merge_rollups(merged_rollups)
+        return nil if merged_rollups.empty?
+
+        combined = MergedRollup.new(0, 0, 0, 0, 0, 0, 0.0, 0.0, nil, nil, {})
+        merged_rollups.each do |rollup|
+          combined.count += rollup.count
+          combined.error_count += rollup.error_count
+          combined.status_2xx += rollup.status_2xx
+          combined.status_3xx += rollup.status_3xx
+          combined.status_4xx += rollup.status_4xx
+          combined.status_5xx += rollup.status_5xx
+          combined.sum_duration += rollup.sum_duration
+          combined.sum_square_duration += rollup.sum_square_duration
+          combined.min_duration = [ combined.min_duration, rollup.min_duration ].compact.min
+          combined.max_duration = [ combined.max_duration, rollup.max_duration ].compact.max
+          rollup.histogram_counts.each do |index, n|
+            combined.histogram_counts[index] = combined.histogram_counts.fetch(index, 0) + n
+          end
+        end
+        combined
+      end
+
       # One read serves the overall request row and every route's. route_id
       # and duration are NOT NULL on requests, so ordering by (route_id,
       # duration) hands each route its durations already sorted.

data/app/views/rails_pulse/components/_panel.html.erb

@@ -2,6 +2,9 @@
   title ||= nil
   help_heading ||= nil
   help_text ||= nil
+  # { heading:, lines: [...] } — a note about the panel's data, shown from
+  # an info icon
+  info ||= nil
   link_to ||= nil
   actions ||= nil
   card_classes ||= ""
@@ -27,6 +30,26 @@
             data-action="click->rails-pulse--series-toggle#toggle"
           ><%= toggle[:name] %></button>
         <% end %>
+        <% if info %>
+          <div data-controller="rails-pulse--popover" data-rails-pulse--popover-placement-value="bottom-end">
+            <a href="#"
+              title="<%= info[:heading] %>"
+              data-rails-pulse--popover-target="button"
+              data-action="rails-pulse--popover#toggle"
+              data-popovertarget="popover">
+              <%= rails_pulse_icon 'info', height: "20px" %>
+            </a>
+
+            <div popover class="popover card" data-rails-pulse--popover-target="menu" style="max-width: 20rem">
+              <div class="flex flex-col">
+                <h3 class="font-semibold leading-none mbe-2 uppercase text-sm"><%= info[:heading] %></h3>
+                <% info[:lines].each_with_index do |line, index| %>
+                  <p class="text-sm text-subtle<%= ' mbe-2' if index < info[:lines].size - 1 %>"><%= line %></p>
+                <% end %>
+              </div>
+            </div>
+          </div>
+        <% end %>
         <% if help_heading %>
           <div data-controller="rails-pulse--popover" data-rails-pulse--popover-placement-value="bottom-start">
             <a href="#"

data/app/views/rails_pulse/queries/_diagnostics.html.erb

@@ -1,5 +1,8 @@
 <%= render 'rails_pulse/components/panel', {
   title: 'Diagnostics',
+  # Built, like the locations and raw data below, from operation rows, which
+  # folded requests never create.
+  info: detail_capture_info("Diagnostics, locations and raw data come only from captured requests: slow, with a slow query, errored or sampled. The charts and cards above count all traffic.", subject: query),
   help_heading: 'Query Diagnostics',
   help_text: 'Automatically generated analysis for this query. Includes N+1 detection (from live request data), query characteristics, detected issues, optimization suggestions, and a database EXPLAIN plan. Click Re-analyze to regenerate.',
   actions: [ { url: reanalyze_query_path(query), method: :post, title: 'Re-analyze', icon: 'refresh-cw' } ]

data/app/views/rails_pulse/requests/index.html.erb

@@ -1,6 +1,9 @@
 <%= render 'rails_pulse/components/page_header', show_active_filters: true %>
 
-<%= render 'rails_pulse/components/panel', { title: 'Requests', } do %>
+<%= render 'rails_pulse/components/panel', {
+  title: 'Requests',
+  info: detail_capture_info("This list shows only captured requests: slow, with a slow query, errored or sampled. Charts and totals count all traffic.")
+} do %>
   <%= search_form_for @ransack_query, url: requests_path, class: "flex items-center justify-between gap mb-4" do |form| %>
     <div class="flex items-center grow gap">
       <%= form.search_field :route_path_cont,

data/app/views/rails_pulse/routes/show.html.erb

@@ -14,6 +14,7 @@
   <div class="grid-item">
     <%= render 'rails_pulse/components/panel', {
       title: 'Route Requests',
+      info: detail_capture_info("This list shows only captured requests: slow, with a slow query, errored or sampled. Charts and totals count all traffic.", subject: @route),
       help_heading: 'Route Performance Charts',
       help_text: 'Toggle between different performance metrics for this route. Response Time shows P95/P99 latency, Request Rate shows traffic patterns, and Error Rate tracks failure rates over time.'
     } do %>

data/app/views/rails_pulse/storage/show.html.erb

@@ -92,6 +92,23 @@
       </dd>
       <dt>Count caps</dt>
       <dd>Configured in <code>max_table_records</code>. Tables without a key have no count limit.</dd>
+      <dt>Full details</dt>
+      <dd>
+        <% detail_capture = @status.detail_capture %>
+        <% if !detail_capture[:enabled] %>
+          All requests are stored in full
+        <% else %>
+          <%= detail_capture[:settings_label].upcase_first %>
+          <br>
+          <span class="text-subtle">
+            <% if (rate = detail_capture[:rate]) %>
+              Stored <%= rate.share_label %> of the last day's <%= number_with_delimiter(rate.total) %> requests in full
+            <% else %>
+              Share kept appears once SummaryJob has summarized an hour with folded requests
+            <% end %>
+          </span>
+        <% end %>
+      </dd>
     </dl>
   <% end %>
 

data/db/rails_pulse_migrate/20261007000001_create_rails_pulse_rollups.rb

@@ -0,0 +1,35 @@
+class CreateRailsPulseRollups < ActiveRecord::Migration[7.0]
+  def change
+    return if table_exists?(:rails_pulse_rollups)
+
+    create_table :rails_pulse_rollups do |t|
+      # Per-writer per-hour aggregates of the requests and SQL operations that
+      # config.store_full_details folded instead of persisting
+      t.references :rollupable, polymorphic: true, null: false, index: false, comment: "Link to Route or Query"
+      t.datetime :period_start, null: false, comment: "Start of the hour the folded requests occurred in"
+      t.string :writer_id, null: false, comment: "host:pid:uuid of the writer incarnation that owns the row"
+
+      t.integer :count, default: 0, null: false, comment: "Folded requests (route rows) or operations (query rows)"
+      t.integer :error_count, default: 0, null: false, comment: "Folded 5xx requests; always 0 on query rows"
+      t.integer :status_2xx, default: 0, null: false, comment: "Folded 2xx responses; always 0 on query rows"
+      t.integer :status_3xx, default: 0, null: false, comment: "Folded 3xx responses; always 0 on query rows"
+      t.integer :status_4xx, default: 0, null: false, comment: "Folded 4xx responses; always 0 on query rows"
+      t.integer :status_5xx, default: 0, null: false, comment: "Folded 5xx responses; always 0 on query rows"
+
+      t.decimal :sum_duration, precision: 20, scale: 6, default: 0, null: false, comment: "Exact sum of folded durations in milliseconds"
+      t.decimal :sum_square_duration, precision: 30, scale: 6, default: 0, null: false, comment: "Exact sum of squared durations, for pooled stddev"
+      t.decimal :min_duration, precision: 15, scale: 6, comment: "Smallest folded duration in milliseconds"
+      t.decimal :max_duration, precision: 15, scale: 6, comment: "Largest folded duration in milliseconds"
+      t.text :histogram, comment: "JSON of DurationHistogram bucket index => count, for percentiles"
+
+      t.timestamps
+    end
+
+    # upsert_all relies on MySQL resolving conflicts through any unique key,
+    # so this must stay the table's only unique index besides the primary key.
+    add_index :rails_pulse_rollups, [ :rollupable_type, :rollupable_id, :period_start, :writer_id ],
+      unique: true,
+      name: "idx_pulse_rollups_unique"
+    add_index :rails_pulse_rollups, :period_start, name: "index_rails_pulse_rollups_on_period_start"
+  end
+end

data/db/rails_pulse_schema.rb

@@ -7,7 +7,7 @@ require "rails_pulse/route_indexes" unless defined?(RailsPulse::RouteIndexes)
 RailsPulse::Schema = lambda do |connection|
   adapter = connection.adapter_name.downcase
   # Skip if all tables already exist to prevent conflicts
-  required_tables = [ :rails_pulse_routes, :rails_pulse_queries, :rails_pulse_requests, :rails_pulse_operations, :rails_pulse_jobs, :rails_pulse_job_runs, :rails_pulse_summaries, :rails_pulse_deployments, :rails_pulse_exception_groups, :rails_pulse_exception_occurrences, :rails_pulse_events ]
+  required_tables = [ :rails_pulse_routes, :rails_pulse_queries, :rails_pulse_requests, :rails_pulse_operations, :rails_pulse_jobs, :rails_pulse_job_runs, :rails_pulse_summaries, :rails_pulse_deployments, :rails_pulse_exception_groups, :rails_pulse_exception_occurrences, :rails_pulse_events, :rails_pulse_rollups ]
 
   # Check which tables already exist
   existing_tables = required_tables.select { |table| connection.table_exists?(table) }
@@ -245,7 +245,7 @@ RailsPulse::Schema = lambda do |connection|
                    comment: "FK to the group this occurrence belongs to"
       t.string   :exception_class, null: false
       t.text     :message
-        t.text     :backtrace,       comment: "JSON array of {file, line, method} frames (first 50)"
+      t.text     :backtrace,       comment: "JSON array of {file, line, method} frames (first 50)"
       t.string   :request_url,     comment: "Nullable — web requests only"
       t.string   :request_method,  comment: "GET, POST, etc."
       t.string   :environment,     comment: "production, staging, etc."
@@ -277,6 +277,38 @@ RailsPulse::Schema = lambda do |connection|
       name: "index_rp_events_on_kind_subject_and_occurred_at"
   end
 
+  unless connection.table_exists?(:rails_pulse_rollups)
+    connection.create_table :rails_pulse_rollups do |t|
+      # Per-writer per-hour aggregates of the requests and SQL operations that
+      # config.store_full_details folded instead of persisting
+      t.references :rollupable, polymorphic: true, null: false, index: false, comment: "Link to Route or Query"
+      t.datetime :period_start, null: false, comment: "Start of the hour the folded requests occurred in"
+      t.string :writer_id, null: false, comment: "host:pid:uuid of the writer incarnation that owns the row"
+
+      t.integer :count, default: 0, null: false, comment: "Folded requests (route rows) or operations (query rows)"
+      t.integer :error_count, default: 0, null: false, comment: "Folded 5xx requests; always 0 on query rows"
+      t.integer :status_2xx, default: 0, null: false, comment: "Folded 2xx responses; always 0 on query rows"
+      t.integer :status_3xx, default: 0, null: false, comment: "Folded 3xx responses; always 0 on query rows"
+      t.integer :status_4xx, default: 0, null: false, comment: "Folded 4xx responses; always 0 on query rows"
+      t.integer :status_5xx, default: 0, null: false, comment: "Folded 5xx responses; always 0 on query rows"
+
+      t.decimal :sum_duration, precision: 20, scale: 6, default: 0, null: false, comment: "Exact sum of folded durations in milliseconds"
+      t.decimal :sum_square_duration, precision: 30, scale: 6, default: 0, null: false, comment: "Exact sum of squared durations, for pooled stddev"
+      t.decimal :min_duration, precision: 15, scale: 6, comment: "Smallest folded duration in milliseconds"
+      t.decimal :max_duration, precision: 15, scale: 6, comment: "Largest folded duration in milliseconds"
+      t.text :histogram, comment: "JSON of DurationHistogram bucket index => count, for percentiles"
+
+      t.timestamps
+    end
+
+    # upsert_all relies on MySQL resolving conflicts through any unique key,
+    # so this must stay the table's only unique index besides the primary key.
+    connection.add_index :rails_pulse_rollups, [ :rollupable_type, :rollupable_id, :period_start, :writer_id ],
+            unique: true,
+            name: "idx_pulse_rollups_unique"
+    connection.add_index :rails_pulse_rollups, :period_start, name: "index_rails_pulse_rollups_on_period_start"
+  end
+
   # Add indexes to existing tables for efficient aggregation
   unless connection.index_exists?(:rails_pulse_requests, [ :created_at, :route_id ], name: "idx_requests_for_aggregation")
     connection.add_index :rails_pulse_requests, [ :created_at, :route_id ], name: "idx_requests_for_aggregation"

data/lib/generators/rails_pulse/base_methods.rb

@@ -16,6 +16,7 @@ module RailsPulse
         rails_pulse_exception_groups
         rails_pulse_exception_occurrences
         rails_pulse_events
+        rails_pulse_rollups
       ].freeze
 
       # Generate next migration number for timestamped migrations

data/lib/generators/rails_pulse/templates/db/rails_pulse_schema.rb

@@ -7,7 +7,7 @@ require "rails_pulse/route_indexes" unless defined?(RailsPulse::RouteIndexes)
 RailsPulse::Schema = lambda do |connection|
   adapter = connection.adapter_name.downcase
   # Skip if all tables already exist to prevent conflicts
-  required_tables = [ :rails_pulse_routes, :rails_pulse_queries, :rails_pulse_requests, :rails_pulse_operations, :rails_pulse_jobs, :rails_pulse_job_runs, :rails_pulse_summaries, :rails_pulse_deployments, :rails_pulse_exception_groups, :rails_pulse_exception_occurrences, :rails_pulse_events ]
+  required_tables = [ :rails_pulse_routes, :rails_pulse_queries, :rails_pulse_requests, :rails_pulse_operations, :rails_pulse_jobs, :rails_pulse_job_runs, :rails_pulse_summaries, :rails_pulse_deployments, :rails_pulse_exception_groups, :rails_pulse_exception_occurrences, :rails_pulse_events, :rails_pulse_rollups ]
 
   # Check which tables already exist
   existing_tables = required_tables.select { |table| connection.table_exists?(table) }
@@ -245,7 +245,7 @@ RailsPulse::Schema = lambda do |connection|
                    comment: "FK to the group this occurrence belongs to"
       t.string   :exception_class, null: false
       t.text     :message
-        t.text     :backtrace,       comment: "JSON array of {file, line, method} frames (first 50)"
+      t.text     :backtrace,       comment: "JSON array of {file, line, method} frames (first 50)"
       t.string   :request_url,     comment: "Nullable — web requests only"
       t.string   :request_method,  comment: "GET, POST, etc."
       t.string   :environment,     comment: "production, staging, etc."
@@ -277,6 +277,38 @@ RailsPulse::Schema = lambda do |connection|
       name: "index_rp_events_on_kind_subject_and_occurred_at"
   end
 
+  unless connection.table_exists?(:rails_pulse_rollups)
+    connection.create_table :rails_pulse_rollups do |t|
+      # Per-writer per-hour aggregates of the requests and SQL operations that
+      # config.store_full_details folded instead of persisting
+      t.references :rollupable, polymorphic: true, null: false, index: false, comment: "Link to Route or Query"
+      t.datetime :period_start, null: false, comment: "Start of the hour the folded requests occurred in"
+      t.string :writer_id, null: false, comment: "host:pid:uuid of the writer incarnation that owns the row"
+
+      t.integer :count, default: 0, null: false, comment: "Folded requests (route rows) or operations (query rows)"
+      t.integer :error_count, default: 0, null: false, comment: "Folded 5xx requests; always 0 on query rows"
+      t.integer :status_2xx, default: 0, null: false, comment: "Folded 2xx responses; always 0 on query rows"
+      t.integer :status_3xx, default: 0, null: false, comment: "Folded 3xx responses; always 0 on query rows"
+      t.integer :status_4xx, default: 0, null: false, comment: "Folded 4xx responses; always 0 on query rows"
+      t.integer :status_5xx, default: 0, null: false, comment: "Folded 5xx responses; always 0 on query rows"
+
+      t.decimal :sum_duration, precision: 20, scale: 6, default: 0, null: false, comment: "Exact sum of folded durations in milliseconds"
+      t.decimal :sum_square_duration, precision: 30, scale: 6, default: 0, null: false, comment: "Exact sum of squared durations, for pooled stddev"
+      t.decimal :min_duration, precision: 15, scale: 6, comment: "Smallest folded duration in milliseconds"
+      t.decimal :max_duration, precision: 15, scale: 6, comment: "Largest folded duration in milliseconds"
+      t.text :histogram, comment: "JSON of DurationHistogram bucket index => count, for percentiles"
+
+      t.timestamps
+    end
+
+    # upsert_all relies on MySQL resolving conflicts through any unique key,
+    # so this must stay the table's only unique index besides the primary key.
+    connection.add_index :rails_pulse_rollups, [ :rollupable_type, :rollupable_id, :period_start, :writer_id ],
+            unique: true,
+            name: "idx_pulse_rollups_unique"
+    connection.add_index :rails_pulse_rollups, :period_start, name: "index_rails_pulse_rollups_on_period_start"
+  end
+
   # Add indexes to existing tables for efficient aggregation
   unless connection.index_exists?(:rails_pulse_requests, [ :created_at, :route_id ], name: "idx_requests_for_aggregation")
     connection.add_index :rails_pulse_requests, [ :created_at, :route_id ], name: "idx_requests_for_aggregation"

data/lib/generators/rails_pulse/templates/rails_pulse.rb

@@ -420,6 +420,19 @@ RailsPulse.configure do |config|
   # Time-based retention - delete records older than this period
   config.full_retention_period = 30.days
 
+  # Which requests are stored in full, with every query and render. :all
+  # stores every request. At high traffic (millions of requests a day), name
+  # a threshold instead: a request is then stored in full only when it is at
+  # or above that request threshold, runs a query at or above that query
+  # threshold, errors, or falls in the random sample_rate_below (default 1%).
+  # Everything else is folded into compact per-route and per-query
+  # aggregates, so dashboard counts, error rates and percentiles still cover
+  # ALL traffic; the requests list then shows the stored requests only.
+  #
+  #   config.store_full_details = { above_threshold: :slow }
+  #   config.store_full_details = { above_threshold: :critical, sample_rate_below: 0.001 }
+  config.store_full_details = :all
+
   # How long rows in rails_pulse_events are kept. The background writer's
   # heartbeats are pruned after a day regardless.
   config.event_retention_period = 90.days

data/lib/rails_pulse/cleanup_service.rb

@@ -109,6 +109,10 @@ module RailsPulse
       run_stage(:time_based, :operations) { cleanup_operations_by_time(cutoff_time) }
       run_stage(:time_based, :job_runs)   { cleanup_job_runs_by_time(cutoff_time) }
       run_stage(:time_based, :requests)   { cleanup_requests_by_time(cutoff_time) }
+      # Rollups substitute for the raw rows detail capture never wrote, and
+      # BackfillSummariesJob reads them, so they live exactly as long as raw
+      # requests do.
+      run_stage(:time_based, :rollups)    { cleanup_rollups_by_time(cutoff_time) } if rollups_table_exists?
       run_stage(:time_based, :queries)    { cleanup_queries_by_time(cutoff_time) }
       run_stage(:time_based, :routes)     { cleanup_routes_by_time(cutoff_time) }
       run_stage(:time_based, :jobs)       { cleanup_jobs_by_time(cutoff_time) }
@@ -195,19 +199,25 @@ module RailsPulse
       )
     end
 
+    # A rollup row covers a whole hour, so only hours that ended before the
+    # cutoff go: the hour the cutoff falls in may not be summarized yet, and
+    # its summary needs every rollup to count the folded traffic.
+    def cleanup_rollups_by_time(cutoff_time)
+      delete_in_batches(
+        RailsPulse::Rollup.where("period_start <= ?", cutoff_time - 1.hour),
+        order_column: :period_start
+      )
+    end
+
     def cleanup_queries_by_time(cutoff_time)
       delete_in_batches(
-        RailsPulse::Query
-          .where("created_at < ?", cutoff_time)
-          .where("NOT EXISTS (SELECT 1 FROM rails_pulse_operations WHERE rails_pulse_operations.query_id = rails_pulse_queries.id)")
+        orphaned_queries_scope.where("created_at < ?", cutoff_time)
       )
     end
 
     def cleanup_routes_by_time(cutoff_time)
       delete_in_batches(
-        RailsPulse::Route
-          .where("created_at < ?", cutoff_time)
-          .where("NOT EXISTS (SELECT 1 FROM rails_pulse_requests WHERE rails_pulse_requests.route_id = rails_pulse_routes.id)")
+        orphaned_routes_scope.where("created_at < ?", cutoff_time)
       )
     end
 
@@ -282,17 +292,42 @@ module RailsPulse
     end
 
     def cleanup_queries_by_count
+      cleanup_by_count(RailsPulse::Query, :rails_pulse_queries, order_column: :created_at, scope: orphaned_queries_scope)
+    end
+
+    def cleanup_routes_by_count
+      cleanup_by_count(RailsPulse::Route, :rails_pulse_routes, order_column: :created_at, scope: orphaned_routes_scope)
+    end
+
+    # A route or query with rollup rows is still feeding hourly summaries
+    # (detail capture folded its traffic instead of persisting it), so it is
+    # not an orphan until its rollups are pruned.
+    def orphaned_queries_scope
       scope = RailsPulse::Query.where(
         "NOT EXISTS (SELECT 1 FROM rails_pulse_operations WHERE rails_pulse_operations.query_id = rails_pulse_queries.id)"
       )
-      cleanup_by_count(RailsPulse::Query, :rails_pulse_queries, order_column: :created_at, scope: scope)
+      if rollups_table_exists?
+        scope = scope.where(
+          "NOT EXISTS (SELECT 1 FROM rails_pulse_rollups WHERE rails_pulse_rollups.rollupable_type = 'RailsPulse::Query' AND rails_pulse_rollups.rollupable_id = rails_pulse_queries.id)"
+        )
+      end
+      scope
     end
 
-    def cleanup_routes_by_count
+    def orphaned_routes_scope
       scope = RailsPulse::Route.where(
         "NOT EXISTS (SELECT 1 FROM rails_pulse_requests WHERE rails_pulse_requests.route_id = rails_pulse_routes.id)"
       )
-      cleanup_by_count(RailsPulse::Route, :rails_pulse_routes, order_column: :created_at, scope: scope)
+      if rollups_table_exists?
+        scope = scope.where(
+          "NOT EXISTS (SELECT 1 FROM rails_pulse_rollups WHERE rails_pulse_rollups.rollupable_type = 'RailsPulse::Route' AND rails_pulse_rollups.rollupable_id = rails_pulse_routes.id)"
+        )
+      end
+      scope
+    end
+
+    def rollups_table_exists?
+      RailsPulse::Rollup.table_available?
     end
 
     def cleanup_jobs_by_count