rails_pulse 0.4.2 → 0.5.0.pre.1

data/app/serializers/rails_pulse/api/v1/job_run_serializer.rb

@@ -0,0 +1,28 @@
+module RailsPulse
+  module Api
+    module V1
+      class JobRunSerializer
+        ERROR_MESSAGE_LIMIT = 500
+
+        def self.serialize(run)
+          {
+            id:            run.id,
+            job_id:        run.job_id,
+            job_name:      run.job&.name,
+            queue_name:    run.job&.queue_name,
+            run_id:        run.run_id,
+            status:        run.status,
+            occurred_at:   run.occurred_at,
+            enqueued_at:   run.enqueued_at,
+            duration:      run.duration,
+            attempts:      run.attempts,
+            adapter:       run.adapter,
+            error_class:   run.error_class,
+            error_message: run.error_message&.truncate(ERROR_MESSAGE_LIMIT),
+            tags:          run.tags
+          }
+        end
+      end
+    end
+  end
+end

data/app/serializers/rails_pulse/api/v1/job_serializer.rb

@@ -0,0 +1,26 @@
+module RailsPulse
+  module Api
+    module V1
+      class JobSerializer
+        # Without `stats` the counters are the job's lifetime totals, cached on
+        # the row. With it they cover one window and come from summaries, so
+        # the two are kept in separate keys rather than one set of names
+        # meaning different things depending on the request.
+        def self.serialize(job, stats: nil)
+          {
+            id:             job.id,
+            name:           job.name,
+            queue_name:     job.queue_name,
+            runs_count:     job.runs_count,
+            failures_count: job.failures_count,
+            avg_duration:   job.avg_duration,
+            p95_duration:   job.p95_duration,
+            p99_duration:   job.p99_duration,
+            failure_rate:   job.failure_rate,
+            stats:          stats
+          }
+        end
+      end
+    end
+  end
+end

data/app/serializers/rails_pulse/api/v1/query_serializer.rb

@@ -0,0 +1,25 @@
+module RailsPulse
+  module Api
+    module V1
+      class QuerySerializer
+        def self.serialize(query, stats: nil)
+          analysis = (query.n_plus_one_analysis || {}).transform_keys(&:to_s)
+
+          {
+            id:             query.id,
+            normalized_sql: query.normalized_sql,
+            hashed_sql:     query.hashed_sql,
+            analyzed_at:    query.analyzed_at,
+            issues:         query.issues,
+            suggestions:    query.suggestions,
+            n_plus_one:     {
+              likely:     analysis["is_likely_n_plus_one"] == true,
+              confidence: analysis["confidence_score"]
+            },
+            stats:          stats
+          }
+        end
+      end
+    end
+  end
+end

data/app/serializers/rails_pulse/api/v1/request_serializer.rb

@@ -0,0 +1,21 @@
+module RailsPulse
+  module Api
+    module V1
+      class RequestSerializer
+        def self.serialize(request)
+          {
+            id:                  request.id,
+            route_id:            request.route_id,
+            occurred_at:         request.occurred_at,
+            duration:            request.duration,
+            status:              request.status,
+            is_error:            request.is_error,
+            request_uuid:        request.request_uuid,
+            controller_action:   request.controller_action,
+            response_size_bytes: request.response_size_bytes
+          }
+        end
+      end
+    end
+  end
+end

data/app/serializers/rails_pulse/api/v1/route_serializer.rb

@@ -0,0 +1,19 @@
+module RailsPulse
+  module Api
+    module V1
+      class RouteSerializer
+        def self.serialize(route, stats: nil)
+          {
+            id:                route.id,
+            http_methods:      route.http_methods_list,
+            path:              route.path,
+            controller_action: route.controller_action,
+            tags:              route.tag_list,
+            created_at:        route.created_at,
+            stats:             stats
+          }
+        end
+      end
+    end
+  end
+end

data/app/services/rails_pulse/exception_capture_service.rb

@@ -55,7 +55,7 @@ module RailsPulse
 
     def capture
       return unless RailsPulse.configuration.track_exceptions
-      return if RequestStore.store[:skip_recording_rails_pulse_activity]
+      return if RailsPulse::Current.skip_recording_rails_pulse_activity
 
       frames      = parse_backtrace(@exception.backtrace || [])
       location    = fingerprint_location(frames)

data/app/services/rails_pulse/operations/comparison.rb

@@ -3,7 +3,7 @@ module RailsPulse
     # The result of measuring a recent window against a historical baseline.
     #
     # This is a value object on purpose. It is the shape callers outside this
-    # namespace — dashboards today, findings and Pro tooling later — are expected
+    # namespace — dashboards today, findings and agent tooling later — are expected
     # to depend on, so it carries every number needed to explain the verdict
     # rather than just the verdict.
     class Comparison

data/app/services/rails_pulse/summary_service/from_child_periods.rb

@@ -0,0 +1,161 @@
+module RailsPulse
+  class SummaryService
+    # Summary rows for a day, week or month, rolled up from the summaries of
+    # its child periods (hours for a day, days for a week or month) without
+    # reading raw rows.
+    #
+    # The database combines the children in one GROUP BY, so Ruby receives
+    # one row per summarizable however many child periods there are. Counts,
+    # totals, averages, min, max, status counts and the standard deviation
+    # combine exactly. P50/P95/P99 are the count-weighted average of the
+    # children's, the same weighting Tables::Base applies when a dashboard
+    # range spans several periods.
+    #
+    # Only plain SUM, MIN, MAX and arithmetic are used, so the query runs
+    # unchanged on SQLite (which has no STDDEV or, without its math
+    # extension, POWER), PostgreSQL and MySQL. Divisions happen in Ruby,
+    # where integer sums cannot truncate them.
+    #
+    # Rows carry the summarizable and its metrics; SummaryService adds the
+    # period columns.
+    class FromChildPeriods
+      TABLE = "rails_pulse_summaries".freeze
+      PERCENTILES = %i[p50_duration p95_duration p99_duration].freeze
+      STATUS_COLUMNS = %i[error_count success_count status_2xx status_3xx status_4xx status_5xx].freeze
+
+      AGGREGATES = {
+        count: "SUM(#{TABLE}.count)",
+        total_duration: "SUM(#{TABLE}.total_duration)",
+        min_duration: "MIN(#{TABLE}.min_duration)",
+        max_duration: "MAX(#{TABLE}.max_duration)",
+        **PERCENTILES.flat_map { |column|
+          [
+            [ :"#{column}_weighted", "SUM(#{TABLE}.#{column} * #{TABLE}.count)" ],
+            [ :"#{column}_weight", "SUM(CASE WHEN #{TABLE}.#{column} IS NOT NULL THEN #{TABLE}.count ELSE 0 END)" ]
+          ]
+        }.to_h,
+        # The three sums Statistics.pooled_stddev needs.
+        stddev_within_sum: "SUM((#{TABLE}.count - 1) * COALESCE(#{TABLE}.stddev_duration, 0) * COALESCE(#{TABLE}.stddev_duration, 0))",
+        weighted_mean_sum: "SUM(#{TABLE}.count * #{TABLE}.avg_duration)",
+        weighted_square_sum: "SUM(#{TABLE}.count * #{TABLE}.avg_duration * #{TABLE}.avg_duration)",
+        **STATUS_COLUMNS.to_h { |column| [ column, "SUM(#{TABLE}.#{column})" ] }
+      }.freeze
+
+      # `child_starts` are the boundaries the children must start on. Rows in
+      # the range that start elsewhere were written under a different
+      # aggregation time zone; they overlap these and would be counted twice.
+      def initialize(child_period_type, child_starts)
+        @child_period_type = child_period_type
+        @child_starts = child_starts
+      end
+
+      # Always exactly one row, even for an empty period (see SummaryService).
+      def request_rows
+        combined = combined_of("RailsPulse::Request")[0]
+
+        [ row("RailsPulse::Request", 0).merge(duration_metrics(combined), status_metrics(combined)) ]
+      end
+
+      def route_rows
+        combined_of("RailsPulse::Route").map do |route_id, combined|
+          row("RailsPulse::Route", route_id).merge(duration_metrics(combined), status_metrics(combined))
+        end
+      end
+
+      def query_rows
+        combined_of("RailsPulse::Query").map do |query_id, combined|
+          row("RailsPulse::Query", query_id).merge(duration_metrics(combined))
+        end
+      end
+
+      def job_rows
+        combined_by_job = combined_of("RailsPulse::Job")
+        return [] if combined_by_job.empty?
+
+        # A job whose row has gone (count-based cleanup) has nothing to
+        # summarize against.
+        known_job_ids = Job.where(id: combined_by_job.keys).pluck(:id).to_set
+
+        combined_by_job.filter_map do |job_id, combined|
+          next unless known_job_ids.include?(job_id)
+
+          row("RailsPulse::Job", job_id).merge(
+            duration_metrics(combined),
+            status_metrics(combined).slice(:error_count, :success_count)
+          )
+        end
+      end
+
+      # The children already carry the all-groups row (id 0), which sums like
+      # any other group.
+      def exception_rows
+        combined_of("RailsPulse::ExceptionGroup").map do |group_id, combined|
+          row("RailsPulse::ExceptionGroup", group_id).merge(count: combined[:count].to_i)
+        end
+      end
+
+      private
+
+      def row(summarizable_type, summarizable_id)
+        { summarizable_type: summarizable_type, summarizable_id: summarizable_id }
+      end
+
+      # { summarizable_type => { summarizable_id => { aggregate => value } } }.
+      # Empty children (an idle hour's count-0 heartbeat) are left out; they
+      # add nothing, and an idle period falls back to empty metrics.
+      def combined_children
+        @combined_children ||= Summary
+          .where(period_type: @child_period_type, period_start: @child_starts)
+          .where(count: 1..)
+          .group(:summarizable_type, :summarizable_id)
+          .pluck(:summarizable_type, :summarizable_id, *AGGREGATES.values.map { |sql| Arel.sql(sql) })
+          .each_with_object({}) do |(type, id, *values), by_type|
+            (by_type[type] ||= {})[id] = AGGREGATES.keys.zip(values).to_h
+          end
+      end
+
+      # { summarizable_id => combined } for one summarizable type.
+      def combined_of(summarizable_type)
+        combined_children.fetch(summarizable_type, {})
+      end
+
+      def duration_metrics(combined)
+        return Metrics.duration([]) unless combined
+
+        count = combined[:count].to_i
+        total = combined[:total_duration].to_f
+
+        {
+          count: count,
+          # Count-weighted mean of the children's averages: identical to
+          # total / count where every row has a duration (requests, routes,
+          # queries), and consistent with the children for jobs, whose count
+          # includes runs with no recorded duration (discarded runs) that
+          # total / count would dilute the average with.
+          avg_duration: combined[:weighted_mean_sum].to_f / count,
+          min_duration: combined[:min_duration]&.to_f,
+          max_duration: combined[:max_duration]&.to_f,
+          total_duration: total,
+          **PERCENTILES.to_h { |column| [ column, weighted_percentile(combined, column) ] },
+          stddev_duration: Statistics.pooled_stddev(
+            count: count,
+            within_sum: combined[:stddev_within_sum],
+            weighted_mean_sum: combined[:weighted_mean_sum],
+            weighted_square_sum: combined[:weighted_square_sum]
+          )
+        }
+      end
+
+      def weighted_percentile(combined, column)
+        weight = combined[:"#{column}_weight"].to_f
+        return nil if weight.zero?
+
+        combined[:"#{column}_weighted"].to_f / weight
+      end
+
+      def status_metrics(combined)
+        STATUS_COLUMNS.index_with { |column| combined ? combined[column].to_i : 0 }
+      end
+    end
+  end
+end

data/app/services/rails_pulse/summary_service/from_raw_rows.rb

@@ -0,0 +1,115 @@
+module RailsPulse
+  class SummaryService
+    # Summary rows for an hour, computed from raw requests, operations, job
+    # runs and exception occurrences.
+    #
+    # Each kind is read with one query, sorted by summarizable and then
+    # duration so every summarizable's durations arrive as one consecutive,
+    # already-sorted run; the overall request row and the per-route rows
+    # share a single read. Memory is bounded by one hour's rows, and the
+    # hour costs one query per kind however many routes, queries or jobs it
+    # saw. Percentiles are exact.
+    #
+    # Rows carry the summarizable and its metrics; SummaryService adds the
+    # period columns.
+    class FromRawRows
+      def initialize(time_range)
+        @time_range = time_range
+      end
+
+      # Always exactly one row, even for an empty hour (see SummaryService).
+      def request_rows
+        durations = request_data.map { |_, duration, _| duration }.compact.sort
+        statuses = request_data.map { |_, _, status| status }
+
+        [ row("RailsPulse::Request", 0).merge(Metrics.duration(durations), Metrics.status(statuses)) ]
+      end
+
+      def route_rows
+        request_data.slice_when { |previous, current| previous.first != current.first }.map do |rows|
+          durations = rows.map { |_, duration, _| duration }
+          statuses = rows.map { |_, _, status| status }
+
+          row("RailsPulse::Route", rows.first.first).merge(
+            Metrics.duration(durations),
+            Metrics.status(statuses),
+            count: rows.size
+          )
+        end
+      end
+
+      def query_rows
+        rows_by_summarizable(Operation.where(occurred_at: @time_range), :query_id).filter_map do |query_id, rows|
+          durations = rows.map(&:first).compact
+          next if durations.empty?
+
+          row("RailsPulse::Query", query_id).merge(Metrics.duration(durations))
+        end
+      end
+
+      def job_rows
+        scope = JobRun.where(occurred_at: @time_range).where(status: JobRun::FINAL_STATUSES)
+        runs_by_job = rows_by_summarizable(scope, :job_id, :status)
+        return [] if runs_by_job.empty?
+
+        # A run whose job row has gone (count-based cleanup) has nothing to
+        # summarize against.
+        known_job_ids = Job.where(id: runs_by_job.map(&:first)).pluck(:id).to_set
+
+        runs_by_job.filter_map do |job_id, runs|
+          next unless known_job_ids.include?(job_id)
+
+          durations = runs.map(&:first).compact.map(&:to_f)
+          next if durations.empty?
+
+          statuses = runs.map(&:second)
+          row("RailsPulse::Job", job_id).merge(
+            Metrics.duration(durations),
+            count: runs.size,
+            error_count: statuses.count { |s| s != "success" },
+            success_count: statuses.count { |s| s == "success" }
+          )
+        end
+      end
+
+      # Per-group counts plus an all-groups row (id 0), so the dashboard can
+      # chart total exception volume without loading one series per group.
+      def exception_rows
+        counts = ExceptionOccurrence.where(occurred_at: @time_range).group(:exception_group_id).count
+        return [] if counts.empty?
+
+        rows = counts.map { |group_id, occurrences| row("RailsPulse::ExceptionGroup", group_id).merge(count: occurrences) }
+        rows << row("RailsPulse::ExceptionGroup", 0).merge(count: counts.values.sum)
+      end
+
+      private
+
+      def row(summarizable_type, summarizable_id)
+        { summarizable_type: summarizable_type, summarizable_id: summarizable_id }
+      end
+
+      # One read serves the overall request row and every route's. route_id
+      # and duration are NOT NULL on requests, so ordering by (route_id,
+      # duration) hands each route its durations already sorted.
+      def request_data
+        @request_data ||= Request
+          .where(occurred_at: @time_range)
+          .order(:route_id, :duration)
+          .pluck(:route_id, :duration, :status)
+      end
+
+      # [[summarizable_id, [[duration, *extra_columns], ...]], ...], with
+      # each summarizable's rows in ascending duration order (NULL durations
+      # land at one end or the other depending on the database; callers
+      # compact them).
+      def rows_by_summarizable(scope, column, *extra_columns)
+        scope
+          .where.not(column => nil)
+          .order(column, :duration)
+          .pluck(column, :duration, *extra_columns)
+          .slice_when { |previous, current| previous.first != current.first }
+          .map { |rows| [ rows.first.first, rows.map { |row| row.drop(1) } ] }
+      end
+    end
+  end
+end

data/app/services/rails_pulse/summary_service/metrics.rb

@@ -0,0 +1,38 @@
+module RailsPulse
+  class SummaryService
+    # The duration and status columns of a summary row, computed from raw
+    # values.
+    module Metrics
+      module_function
+
+      # `durations` must be sorted. Empty input yields count 0 and nil
+      # percentiles, which is what an idle period should record.
+      def duration(durations)
+        avg = durations.any? ? durations.sum.to_f / durations.size : 0
+
+        {
+          count: durations.size,
+          avg_duration: avg,
+          min_duration: durations.first,
+          max_duration: durations.last,
+          total_duration: durations.sum,
+          p50_duration: Statistics.calculate_percentile(durations, 0.5),
+          p95_duration: Statistics.calculate_percentile(durations, 0.95),
+          p99_duration: Statistics.calculate_percentile(durations, 0.99),
+          stddev_duration: Statistics.calculate_stddev(durations, avg)
+        }
+      end
+
+      def status(statuses)
+        {
+          error_count: statuses.count { |s| s >= 500 },
+          success_count: statuses.count { |s| s < 500 },
+          status_2xx: statuses.count { |s| s.between?(200, 299) },
+          status_3xx: statuses.count { |s| s.between?(300, 399) },
+          status_4xx: statuses.count { |s| s.between?(400, 499) },
+          status_5xx: statuses.count { |s| s >= 500 }
+        }
+      end
+    end
+  end
+end