rails_pulse 0.5.0 → 0.5.1.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3a4671cfd97c14be6772995e98269ab8371d16ab80143d3ae3e79cebb7c82ff
-  data.tar.gz: 557942447466d4b8b555b9c398cea09ecf39a5f18cc604acc972ec25f1399abd
+  metadata.gz: '08538cb7f42c80b0e0cc87f03be930ebda1242bcaec8e3a95f6e6c2fd4894b6e'
+  data.tar.gz: a442604b862d2ed178ef9636a17529e285bdf1795d13140b7f1003591ebd7f93
 SHA512:
-  metadata.gz: 10a91ca1ebba8d5baac05e287c1582fa6b98274d36b4666bdd52cea2fd9df7c7cc9724c6c2e43c408725f366761a1be4c0dc1c008aea2d3f129e9e9470fa182e
-  data.tar.gz: 2e6d6aadae30d286aceef73717424b3f850ea8ed07b004fd6d6e4a40c431fc6516d0708fd63375598de5bbc4fb56cd775513947274bf13baaea8d19e9b678ab7
+  metadata.gz: 52ecffc7297e1ce526134121dd7a89d401fe4569939c6cfdd3151f4c95043062c9c29416668ddea11bb132a5205ba4ff41c02915a5a990365175911a23434450
+  data.tar.gz: 13454d0ce774b2a0b6df138cc328329265968f67b1a5faa3164a90dea173481c8dce0a7fc505f5c140e6875991ef543baebb8a027fb3222c447b8eab6df3c25a

data/CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.5.1.pre.1] - 2026-10-21
+
+### Added
+
+- **Detail capture for high-traffic installs.** With `config.store_full_details = { above_threshold: :slow }`, only requests and queries past the named threshold, errors and a 1% sample are stored in full, while dashboards still count all traffic from compact hourly aggregates. Off by default; upgrading adds a table, so run `rails generate rails_pulse:upgrade && rails db:migrate`.
+- **How much detail capture keeps.** `rails rails_pulse:status`, the Storage page and the info icon on lists of captured requests show what share of the last day's traffic was kept in full; when it is over a quarter, the dashboard's Storage badge warns and says which setting to change.
+
+### Fixed
+
+- Requests whose controller raises an unhandled exception are now recorded, with the status the error page sends, so error rates and request counts include them; previously only the exception was tracked and the request was missing.
+
 ## [0.5.0] - 2026-10-16
 
 ## [0.5.0.pre.2] - 2026-10-11

data/app/controllers/rails_pulse/api/v1/base_controller.rb

@@ -127,6 +127,15 @@ module RailsPulse
           data = collection.limit(limit).offset(offset)
           [ data, { total: total, limit: limit, offset: offset } ]
         end
+
+        # Rows and counts read straight from raw requests or operations cover
+        # only the captured subset while detail capture is on (decision 0022);
+        # the flag tells API consumers so. Aggregates from summaries stay
+        # complete and carry no flag.
+        def apply_sampled_detail(meta)
+          meta[:sampled_detail] = true if RailsPulse.configuration.detail_capture_settings
+          meta
+        end
       end
     end
   end

data/app/controllers/rails_pulse/api/v1/coverage_controller.rb

@@ -14,13 +14,16 @@ module RailsPulse
         SUMMARY_GRACE = 2.hours
 
         def show
-          render json: {
+          payload = {
             as_of: Time.current.utc.iso8601,
             telemetry: telemetry,
             summaries: summaries,
             retention: retention,
             collection: collection
           }
+          # Raw request counts cover only captured requests under detail capture
+          payload[:sampled_detail] = true if RailsPulse.configuration.detail_capture_settings
+          render json: payload
         end
 
         private

data/app/controllers/rails_pulse/api/v1/queries_controller.rb

@@ -65,7 +65,7 @@ module RailsPulse
             QuerySerializer.serialize(query, stats: stats_for(row, locations[row.query_id])) if query
           end
 
-          render json: { data: data, meta: { total: total, limit: limit, offset: offset } }
+          render json: { data: data, meta: apply_sampled_detail({ total: total, limit: limit, offset: offset }) }
         end
 
         # Restricts the operations to those issued while serving one route, so

data/app/controllers/rails_pulse/api/v1/requests_controller.rb

@@ -20,7 +20,7 @@ module RailsPulse
           return if performed?
 
           data, meta = paginated(collection)
-          render json: { data: data.map { |request| RequestSerializer.serialize(request) }, meta: meta }
+          render json: { data: data.map { |request| RequestSerializer.serialize(request) }, meta: apply_sampled_detail(meta) }
         end
 
         private

data/app/controllers/rails_pulse/api/v1/routes_controller.rb

@@ -83,7 +83,7 @@ module RailsPulse
             meta[:routes_with_traffic] = routes_with_traffic
           end
 
-          render json: { data: data, meta: meta }
+          render json: { data: data, meta: apply_sampled_detail(meta) }
         end
 
         def min_requests

data/app/helpers/rails_pulse/application_helper.rb

@@ -4,6 +4,7 @@ module RailsPulse
     include BreadcrumbsHelper
     include ChartHelper
     include CspHelper
+    include DetailCaptureHelper
     include FormattingHelper
     include FormHelper
     include IconHelper

data/app/helpers/rails_pulse/detail_capture_helper.rb

@@ -0,0 +1,43 @@
+module RailsPulse
+  module DetailCaptureHelper
+    # The panel info (see components/_panel) for a panel built from
+    # individual requests while store_full_details names a threshold: those rows are the
+    # captured subset, but every aggregate still counts all traffic
+    # (decision 0022). nil while store_full_details is :all.
+    #
+    # subject is the route or query the page is about, or nil for all requests.
+    def detail_capture_info(text, subject: nil)
+      return unless RailsPulse.configuration.detail_capture_settings
+
+      rate = DetailCaptureRate.for(subject)
+      lines = [ text, detail_capture_rate_sentence(rate, subject) ]
+      lines << detail_capture_retune_sentence if detail_capture_rate_high?(rate, subject)
+
+      { heading: "Full details", lines: lines.compact }
+    end
+
+    private
+
+    # Only the overall share says the settings need retuning: a route that is
+    # always slow is meant to be captured in full.
+    def detail_capture_rate_high?(rate, subject = nil)
+      subject.nil? && rate.present? && rate.high?
+    end
+
+    def detail_capture_retune_sentence
+      advice = DetailCaptureRate.retune_advice(RailsPulse.configuration.detail_capture_settings)
+      "That's over a quarter of them, so little storage is saved. To store less, #{advice}."
+    end
+
+    def detail_capture_rate_sentence(rate, subject)
+      return unless rate
+
+      noun = case subject
+      when Route then "of this route's requests"
+      when Query then "of this query's executions"
+      else "of all requests"
+      end
+      "Captured in the last day: #{rate.share_label} #{noun}."
+    end
+  end
+end

data/app/jobs/rails_pulse/summary_job.rb

@@ -6,9 +6,16 @@ module RailsPulse
     def perform(target_hour = nil)
       target_hour ||= 1.hour.ago.beginning_of_hour
 
+      # Rollup buckets flush on their own clock, so make this process's
+      # folded traffic visible before the hour is read. Other processes'
+      # buckets are healed by resummarize_hours_with_late_rollups next run.
+      Tracker.flush_rollups!
+
       # Always run hourly summary
       process_hourly_summary(target_hour)
 
+      resummarize_hours_with_late_rollups(target_hour)
+
       # Check if we should run daily summary (at the start of a new day)
       if midnight?(target_hour)
         process_daily_summary(target_hour.to_date - 1.day)
@@ -26,8 +33,46 @@ module RailsPulse
       raise
     end
 
+    # A bucket stops accepting folds STALE_HOUR_GRACE past its hour and is
+    # flushed within FLUSH_INTERVAL, so nothing lands later than this after
+    # the hour it counts.
+    LATE_ROLLUP_LOOKBACK = 3.hours
+
     private
 
+    # Rollups flush about once a minute per process, and a long request that
+    # started in an hour folds only after it finishes, so rows for an hour can
+    # land after that hour was summarized — summaries would otherwise
+    # permanently undercount each hour's folded tail. A recent hour whose
+    # summary no longer matches its raw request count plus its folded route
+    # counts is summarized again; the upsert makes that idempotent. Counts are
+    # compared rather than timestamps because the summary's updated_at is
+    # written by the database and rollup timestamps by Ruby, which clock skew
+    # can order either way. Runs before the daily rollup so a corrected hour
+    # feeds its day. Healing is best-effort: a failure here must not stop the
+    # current hour's summaries or the period rollups.
+    def resummarize_hours_with_late_rollups(target_hour)
+      return unless RailsPulse::Rollup.table_available?
+
+      window = (target_hour - LATE_ROLLUP_LOOKBACK)...target_hour
+      hours = RailsPulse::Rollup.where(period_start: window).distinct.pluck(:period_start)
+
+      hours.each do |hour|
+        summarized_count = RailsPulse::Summary.overall_requests
+          .where(period_type: "hour", period_start: hour).pick(:count)
+        next unless summarized_count
+
+        current_count = RailsPulse::Request.where(occurred_at: hour...(hour + 1.hour)).count +
+          RailsPulse::Rollup.where(rollupable_type: "RailsPulse::Route", period_start: hour).sum(:count)
+        next if current_count == summarized_count
+
+        RailsPulse.logger.info "Re-summarizing hour #{hour}: rollups or requests landed after its summary"
+        SummaryService.new("hour", hour).perform
+      end
+    rescue StandardError => e
+      RailsPulse.logger.error "Could not re-summarize hours with late rollups: #{e.class} - #{e.message}"
+    end
+
     # Cloud receives an hour only once it is summarized, so the sync follows
     # this job rather than running on a schedule of its own. A queue that
     # refuses the job must not fail the summaries already written.

data/app/models/rails_pulse/dashboard/storage_pressure.rb

@@ -11,7 +11,16 @@ module RailsPulse
       # Memoized: computing the items costs several aggregate queries (one a
       # raw-request count), and one dashboard render asks for them repeatedly.
       def pressure_items
-        @pressure_items ||= summary_staleness_items + sub_hour_retention_items + writer_drop_items
+        @pressure_items ||= summary_staleness_items + sub_hour_retention_items + writer_drop_items + detail_capture_items
+      end
+
+      # Memoized (nil is a valid value: detail capture off or no rate yet) and
+      # shared with StorageStatus, which wraps this object — one storage page
+      # render would otherwise compute the rate's aggregate queries repeatedly.
+      def detail_capture_rate
+        return @detail_capture_rate if defined?(@detail_capture_rate)
+
+        @detail_capture_rate = RailsPulse::DetailCaptureRate.for
       end
 
       # Dropped requests get their own Tracking badge, not this one
@@ -103,6 +112,39 @@ module RailsPulse
         []
       end
 
+      # Signal D — store_full_details keeping over a quarter of traffic, so the
+      # requests and operations tables grow almost as if it were :all
+      def detail_capture_items
+        settings = @config.detail_capture_settings
+        return [] unless settings
+
+        rate = detail_capture_rate
+        return [] unless rate&.high?
+
+        routes = RailsPulse::DetailCaptureRate.top_kept_routes_label
+        [ {
+          type:          "STORAGE",
+          name:          "Full details",
+          reason:        "Kept #{rate.share_label} of the last day's requests in full, so little storage is saved",
+          metric:        "#{rate.share_label} kept",
+          metric_sub:    "of #{rate.total.to_fs(:delimited)} requests, last day",
+          link:          "#",
+          severity:      :warning,
+          # Cleanup still runs; store_full_details only decides what it has to delete
+          cleanup:       false,
+          sort_score:    rate.share,
+          popover_title: "Full details are stored for over a quarter of requests",
+          popover_body:  "With <code>config.store_full_details</code> #{ERB::Util.html_escape(settings.description)}, only those requests " \
+                         "are stored in full; the rest are folded into hourly totals. In the last day #{rate.share_label} were stored " \
+                         "in full, so the requests and operations tables are growing almost as if every request were.<br><br>" \
+                         "<strong>How to fix:</strong> #{RailsPulse::DetailCaptureRate.retune_advice(settings).upcase_first} " \
+                         "in the Rails Pulse initializer." +
+                         (routes ? "<br><br>Routes keeping the most: #{ERB::Util.html_escape(routes)}." : "")
+        } ]
+      rescue ActiveRecord::ActiveRecordError
+        []
+      end
+
       def stale_item(severity, reason, metric, metric_sub)
         {
           type:          "STORAGE",

data/app/models/rails_pulse/dashboard/storage_status.rb

@@ -132,6 +132,14 @@ module RailsPulse
         }
       end
 
+      # rate is nil while detail capture is off or its rate is not known yet
+      def detail_capture
+        @detail_capture ||= begin
+          settings = @config.detail_capture_settings
+          { enabled: settings.present?, settings_label: settings&.description, rate: settings && @pressure.detail_capture_rate }
+        end
+      end
+
       def database
         @database ||= {
           adapter: adapter_label,
@@ -514,7 +522,7 @@ module RailsPulse
       end
 
       def cleanup_health
-        items = pressure_items
+        items = pressure_items.reject { |item| item[:cleanup] == false }
         return :critical if items.any? { |item| item[:severity] == :critical }
         return :warning if items.any? { |item| item[:severity] == :warning }
         return :disabled unless @config.archiving_enabled

data/app/models/rails_pulse/operation.rb

@@ -52,25 +52,7 @@ module RailsPulse
     def self.persist_bulk(ops, context)
       return if ops.empty?
 
-      # Phase 1: normalise each unique SQL source once.
-      # norm_cache avoids re-normalising repeated SQL (e.g. N+1 queries).
-      # norm_map feeds the bulk Query resolver: { hashed_sql => normalized_sql }.
-      norm_cache = {}
-      norm_map = {}
-      ops.each do |op|
-        next unless op[:operation_type] == "sql"
-        sql_source = op[:actual_sql].presence || op[:label].presence
-        next unless sql_source
-        unless norm_cache.key?(sql_source)
-          normalized = RailsPulse::SqlQueryNormalizer.normalize(sql_source)
-          hashed = Digest::MD5.hexdigest(normalized)
-          norm_cache[sql_source] = [ hashed, normalized ]
-          norm_map[hashed] ||= normalized
-        end
-      end
-
-      # Phase 2: resolve Query IDs in bulk (1 SELECT in steady state).
-      query_id_map = RailsPulse::Query.bulk_find_or_create(norm_map)
+      norm_cache, query_id_map = normalize_and_resolve(ops)
 
       now = Time.current
       rows = ops.map do |op|
@@ -96,6 +78,50 @@ module RailsPulse
       insert_all!(rows)
     end
 
+    # The (query_id, duration) pairs of a batch's SQL operations, resolving
+    # (or creating) the Query records exactly as persist_bulk would, without
+    # persisting the operations themselves. Detail capture folds these pairs
+    # into per-query rollups so the Queries page keeps counting every request.
+    def self.resolve_sql_durations(ops)
+      ops = Array(ops).select { |op| op[:operation_type] == "sql" }
+      return [] if ops.empty?
+
+      norm_cache, query_id_map = normalize_and_resolve(ops)
+      ops.filter_map do |op|
+        sql_source = op[:actual_sql].presence || op[:label].presence
+        next unless sql_source && (meta = norm_cache[sql_source])
+
+        query_id = query_id_map[meta.first]
+        next unless query_id
+
+        [ query_id, op[:duration].to_f ]
+      end
+    end
+
+    # Phase 1: normalise each unique SQL source once.
+    # norm_cache avoids re-normalising repeated SQL (e.g. N+1 queries) and maps
+    # sql_source => [hashed, normalized]; norm_map feeds the bulk Query
+    # resolver with { hashed_sql => normalized_sql }.
+    # Phase 2: resolve Query IDs in bulk (1 SELECT in steady state).
+    def self.normalize_and_resolve(ops)
+      norm_cache = {}
+      norm_map = {}
+      ops.each do |op|
+        next unless op[:operation_type] == "sql"
+        sql_source = op[:actual_sql].presence || op[:label].presence
+        next unless sql_source
+        unless norm_cache.key?(sql_source)
+          normalized = RailsPulse::SqlQueryNormalizer.normalize(sql_source)
+          hashed = Digest::MD5.hexdigest(normalized)
+          norm_cache[sql_source] = [ hashed, normalized ]
+          norm_map[hashed] ||= normalized
+        end
+      end
+
+      [ norm_cache, RailsPulse::Query.bulk_find_or_create(norm_map) ]
+    end
+    private_class_method :normalize_and_resolve
+
     def self.ransackable_attributes(auth_object = nil)
       %w[id occurred_at label duration start_time average_query_time_ms query_count operation_type query_id]
     end

data/app/models/rails_pulse/rollup.rb

@@ -0,0 +1,50 @@
+module RailsPulse
+  # One writer process's cumulative aggregate of the requests (per route) or
+  # SQL operations (per query) that detail capture folded instead of
+  # persisting during one hour. Decision 0022.
+  #
+  # Rows are written by replace-upsert: a writer incarnation owns the rows
+  # carrying its writer_id and overwrites them with its running totals, so
+  # flushing twice is harmless and two processes never contend. Readers sum
+  # across writer_ids. Sums, counts, min and max are exact; only the duration
+  # histogram (for percentiles) is approximate.
+  class Rollup < RailsPulse::ApplicationRecord
+    self.table_name = "rails_pulse_rollups"
+
+    UNIQUE_INDEX = :idx_pulse_rollups_unique
+
+    belongs_to :rollupable, polymorphic: true
+
+    validates :period_start, :writer_id, presence: true
+
+    scope :for_hour, ->(hour_start) { where(period_start: hour_start) }
+
+    # MySQL has no explicit conflict target and resolves through any unique
+    # key, which is why the model's unique index must stay the only one.
+    def self.upsert_rows(rows)
+      return if rows.empty?
+
+      unique_by = connection.supports_insert_conflict_target? ? UNIQUE_INDEX : nil
+      upsert_all(rows, unique_by: unique_by)
+    end
+
+    def self.ransackable_attributes(_auth_object = nil)
+      []
+    end
+
+    def self.ransackable_associations(_auth_object = nil)
+      []
+    end
+
+    # table_exists?, rescued — for an upgrader who hasn't migrated yet
+    def self.table_available?
+      table_exists?
+    rescue ActiveRecord::ActiveRecordError
+      false
+    end
+
+    def histogram_counts
+      DurationHistogram.load(histogram).counts
+    end
+  end
+end

data/app/services/rails_pulse/detail_capture_rate.rb

@@ -0,0 +1,141 @@
+module RailsPulse
+  # What share of the last day's traffic store_full_details kept in full
+  # (decision 0022), for all requests, one route or one query.
+  #
+  # Reads summaries and rollups only, never the requests table: an hour's
+  # summary counts every request, the rollups count the ones folded, and the
+  # difference was kept. That limits it to hours SummaryJob has summarized.
+  #
+  # The window starts at the first hour with any rollup, so hours from before
+  # store_full_details was set (when everything was kept) don't count. With
+  # no rollup at all in the last day the rate is unknown: it was set too
+  # recently, or nothing is being folded.
+  class DetailCaptureRate
+    WINDOW = 24.hours
+
+    # Above this share, storing only some full details saves little; the
+    # named threshold likely sits below typical timings.
+    HIGH_SHARE = 0.25
+
+    attr_reader :kept, :total
+
+    # subject: nil for all requests, or a Route or Query
+    def self.for(subject = nil, now: Time.current)
+      return unless Rollup.table_available?
+
+      new(subject, now).compute
+    end
+
+    # What to change when the share is high, e.g. "raise
+    # request_thresholds[:slow] (700ms now) or query_thresholds[:slow] (100ms
+    # now), set above_threshold: higher, or lower sample_rate_below (1% now)"
+    def self.retune_advice(settings)
+      key = settings.threshold.inspect
+      advice = "raise request_thresholds[#{key}] (#{settings.requests_slower_than.round}ms now) " \
+               "or query_thresholds[#{key}] (#{settings.queries_slower_than.round}ms now), " \
+               "set store_full_details[:above_threshold] to a higher threshold"
+      return advice unless settings.sample_rate.positive?
+
+      "#{advice}, or lower store_full_details[:sample_rate_below] (#{settings.sample_percent} now)"
+    end
+
+    # "/reports (2,900 of 3,000), /search (400 of 1,200)", or nil when no
+    # route kept anything
+    def self.top_kept_routes_label(now: Time.current)
+      routes = top_kept_routes(now: now).map do |route, kept, total|
+        "#{route.path} (#{kept.to_fs(:delimited)} of #{total.to_fs(:delimited)})"
+      end
+      routes.join(", ").presence
+    end
+
+    # Routes keeping the most requests in the window, as
+    # [[route, kept, total], ...], most kept first.
+    def self.top_kept_routes(limit: 3, now: Time.current)
+      return [] unless Rollup.table_available?
+
+      new(nil, now).top_kept_routes(limit)
+    end
+
+    def initialize(subject, now)
+      @subject = subject
+      @now = now
+    end
+
+    def compute
+      hours = summarized_hours
+      return if hours.empty?
+
+      @total = summaries.where(period_start: hours).sum(:count).to_i
+      return if @total.zero?
+
+      folded = rollups.where(period_start: hours).sum(:count).to_i
+      @kept = [ @total - folded, 0 ].max
+      self
+    end
+
+    def top_kept_routes(limit)
+      hours = summarized_hours(Summary.for_routes)
+      return [] if hours.empty?
+
+      totals = Summary.for_routes.where(period_type: "hour", period_start: hours).group(:summarizable_id).sum(:count)
+      folded = Rollup.where(rollupable_type: "RailsPulse::Route", period_start: hours).group(:rollupable_id).sum(:count)
+      kept = totals.to_h { |id, total| [ id, [ total.to_i - folded[id].to_i, 0 ].max ] }
+
+      top = kept.select { |_id, count| count.positive? }.max_by(limit) { |_id, count| count }
+      routes = Route.where(id: top.map(&:first)).index_by(&:id)
+      top.filter_map { |id, count| [ routes[id], count, totals[id].to_i ] if routes[id] }
+    end
+
+    def share
+      kept.fdiv(total)
+    end
+
+    def high?
+      share > HIGH_SHARE
+    end
+
+    # "17.7%", "under 0.1%"
+    def share_label
+      percent = share * 100
+      return "under 0.1%" if percent.positive? && percent < 0.1
+
+      "#{percent.round(1).to_s.delete_suffix('.0')}%"
+    end
+
+    private
+
+    def window_start
+      @window_start ||= (@now - WINDOW).beginning_of_hour
+    end
+
+    # Rollups for the current hour exist before its summary does, so the
+    # hours come from the summaries.
+    def summarized_hours(scope = subject_summaries)
+      first_folded = Rollup.where(rollupable_type: rollupable_type, period_start: window_start..).minimum(:period_start)
+      return [] unless first_folded
+
+      Summary.where(period_type: "hour", period_start: first_folded..)
+        .merge(scope)
+        .distinct
+        .pluck(:period_start)
+    end
+
+    def summaries
+      Summary.where(period_type: "hour").merge(subject_summaries)
+    end
+
+    def subject_summaries
+      @subject ? Summary.where(summarizable: @subject) : Summary.overall_requests
+    end
+
+    def rollups
+      @subject ? Rollup.where(rollupable: @subject) : Rollup.where(rollupable_type: "RailsPulse::Route")
+    end
+
+    # A route or all requests starts at the first folded request anywhere,
+    # not its own, so a route whose every request was kept still has a rate.
+    def rollupable_type
+      @subject.is_a?(Query) ? "RailsPulse::Query" : "RailsPulse::Route"
+    end
+  end
+end

data/app/services/rails_pulse/operation_suggestions.rb

@@ -1,4 +1,7 @@
 module RailsPulse
+  # Works on persisted operations, which under detail capture are the slow,
+  # errored and sampled requests' — a slow-biased sample, which is the right
+  # input for suggestion tooling.
   class OperationSuggestions
     VIEW_TYPES = %w[template partial layout collection].freeze
 

data/app/services/rails_pulse/query_analysis_service.rb

@@ -1,5 +1,9 @@
 # Orchestrates comprehensive query analysis using modular analyzers.
-# Coordinates multiple specialized analyzers and consolidates results into actionable insights.
+# Coordinates multiple specialized analyzers and consolidates results into
+# actionable insights. Reads persisted operations, which under detail capture
+# come from requests that were slow, ran a slow query, errored or were
+# sampled — a slow-biased sample,
+# which is the right input for analysis that hunts problems.
 module RailsPulse
   class QueryAnalysisService
     def self.analyze_query(query_id)