railswatch 1.0.0

data/lib/railswatch/models/request_record.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Models
+    class RequestRecord < BaseRecord
+      self.table_name = 'railswatch_request_records'
+
+      serialize :custom_data, coder: JSON
+      serialize :backtrace, coder: JSON
+      serialize :request_context, coder: JSON
+
+      attr_accessor :exception_object
+
+      validates :request_id, presence: true
+      validates :occurred_at, presence: true
+
+      def method
+        http_method
+      end
+
+      def method=(value)
+        self.http_method = value
+      end
+
+      def view_runtime
+        view_runtime_ms
+      end
+
+      def view_runtime=(value)
+        self.view_runtime_ms = value
+      end
+
+      def db_runtime
+        db_runtime_ms
+      end
+
+      def db_runtime=(value)
+        self.db_runtime_ms = value
+      end
+
+      def duration
+        duration_ms
+      end
+
+      def duration=(value)
+        self.duration_ms = value
+      end
+
+      def controller_action
+        "#{controller}##{action}"
+      end
+
+      def controller_action_format
+        "#{controller}##{action}|#{format}"
+      end
+
+      def payload_hash
+        {
+          'view_runtime' => view_runtime_ms,
+          'db_runtime' => db_runtime_ms,
+          'duration' => duration_ms,
+          'http_referer' => http_referer,
+          'custom_data' => custom_data,
+          'exception' => exception,
+          'backtrace' => backtrace,
+          'request_context' => request_context
+        }
+      end
+
+      def record_hash
+        base_record_hash.merge(custom_record_hash)
+      end
+
+      def base_record_hash # rubocop:disable Metrics/MethodLength
+        {
+          controller: controller,
+          action: action,
+          format: self.format,
+          status: status,
+          method: http_method,
+          path: path,
+          request_id: request_id,
+          datetime: occurred_at,
+          datetimei: occurred_at.to_i,
+          duration: duration_ms,
+          db_runtime: db_runtime_ms,
+          view_runtime: view_runtime_ms,
+          exception: exception,
+          backtrace: backtrace,
+          http_referer: http_referer,
+          request_context: request_context
+        }
+      end
+
+      def custom_record_hash
+        custom_data.is_a?(Hash) ? custom_data.deep_symbolize_keys : {}
+      end
+
+      before_save do
+        self.exception = Array.wrap(exception).compact.join(' ') if exception.is_a?(Array)
+        self.backtrace = exception_object.backtrace.take(20) if exception_object
+      end
+    end
+  end
+end

data/lib/railswatch/models/resource_record.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Models
+    class ResourceRecord < BaseRecord
+      self.table_name = 'railswatch_resource_records'
+
+      serialize :payload, coder: JSON
+
+      def json
+        payload
+      end
+
+      def json=(value)
+        self.payload = value
+      end
+
+      def payload_hash
+        payload || {}
+      end
+
+      def record_hash
+        payload_hash.symbolize_keys.merge(
+          server: server,
+          role: role,
+          context: context,
+          datetime: occurred_at,
+          datetimei: occurred_at.to_i
+        )
+      end
+    end
+  end
+end

data/lib/railswatch/models/sidekiq_record.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Models
+    class SidekiqRecord < BaseRecord
+      self.table_name = 'railswatch_sidekiq_records'
+
+      serialize :job_args, coder: JSON
+
+      def duration
+        duration_ms
+      end
+
+      def duration=(value)
+        self.duration_ms = value
+      end
+
+      def payload_hash
+        {
+          'message' => message,
+          'duration' => duration_ms
+        }
+      end
+
+      def record_hash # rubocop:disable Metrics/MethodLength
+        {
+          worker: worker,
+          queue: queue,
+          jid: jid,
+          status: status,
+          datetimei: occurred_at.to_i,
+          datetime: Railswatch::Utils.from_datetimei(start_timei.to_i),
+          duration: duration_ms,
+          message: message,
+          job_args: job_args,
+          error_backtrace: error_backtrace
+        }
+      end
+    end
+  end
+end

data/lib/railswatch/models/trace_record.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Models
+    class TraceRecord < BaseRecord
+      self.table_name = 'railswatch_trace_records'
+
+      serialize :entries, coder: JSON
+
+      validates :request_id, presence: true
+
+      def value
+        entries || []
+      end
+
+      def value=(val)
+        self.entries = val
+      end
+    end
+  end
+end

data/lib/railswatch/pruner.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Railswatch
+  class Pruner
+    DEFAULT_BATCH_SIZE = 1_000
+
+    MODEL_TYPES = {
+      requests: 'RequestRecord',
+      sidekiq: 'SidekiqRecord',
+      delayed_job: 'DelayedJobRecord',
+      grape: 'GrapeRecord',
+      rake: 'RakeRecord',
+      custom: 'CustomRecord',
+      resources: 'ResourceRecord',
+      traces: 'TraceRecord',
+      events: 'EventRecord'
+    }.freeze
+
+    def self.call(batch_size: DEFAULT_BATCH_SIZE)
+      MODEL_TYPES.each_with_object({}) do |(type, class_name), results|
+        deleted = deleted_records(type, class_name, batch_size)
+        results[type] = deleted unless deleted.nil?
+      end
+    end
+
+    def self.deleted_records(type, class_name, batch_size)
+      retention = Railswatch.retention[type]
+      return if retention.nil?
+
+      model = Railswatch::Models.const_get(class_name)
+      threshold = Railswatch::Utils.kind_of_now - retention
+
+      delete_in_batches(model, threshold, batch_size)
+    end
+
+    def self.delete_in_batches(model, threshold, batch_size)
+      deleted = 0
+
+      loop do
+        ids = model.where('occurred_at < ?', threshold).limit(batch_size).pluck(:id)
+        break deleted if ids.empty?
+
+        deleted += model.where(id: ids).delete_all
+      end
+    end
+  end
+end

data/lib/railswatch/rails/middleware.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Rails
+    class MiddlewareTraceStorerAndCleanup
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        dup.call!(env)
+      end
+
+      def call!(env)
+        Railswatch.skip = true if monitoring_path?(env)
+
+        @status, @headers, @response = @app.call(env)
+        save_trace_record unless Railswatch.skip
+
+        CurrentRequest.cleanup
+
+        [@status, @headers, @response]
+      end
+
+      private
+
+      def monitoring_path?(env)
+        /#{Railswatch.mount_at}/.match?(env['PATH_INFO'])
+      end
+
+      def save_trace_record
+        Railswatch::Models::TraceRecord.new(
+          request_id: CurrentRequest.current.request_id,
+          value: CurrentRequest.current.tracings,
+          occurred_at: Railswatch::Utils.time
+        ).save
+      end
+    end
+
+    class Middleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        dup.call!(env)
+      end
+
+      def call!(env)
+        @status, @headers, @response = @app.call(env)
+        save_request_record(env)
+
+        [@status, @headers, @response]
+      end
+
+      private
+
+      def save_request_record(env)
+        data = CurrentRequest.current.data
+        return if Railswatch.skip || CurrentRequest.current.ignore.include?(:monitoring) || data.blank?
+
+        record = Railswatch::Models::RequestRecord.new(**data, request_id: CurrentRequest.current.request_id)
+        normalize_status!(record)
+        add_http_referer!(record, env)
+        add_custom_data!(record, env)
+        add_request_context!(record, env)
+        record.save
+      end
+
+      def normalize_status!(record)
+        record.status ||= @status
+        record.status = record.status.to_s if record.status.present?
+      end
+
+      def add_http_referer!(record, env)
+        record.http_referer = env['HTTP_REFERER'] if record.status == '404'
+      end
+
+      def add_custom_data!(record, env)
+        return unless Railswatch.custom_data_proc
+
+        record.custom_data = Railswatch.custom_data_proc.call(env)
+      end
+
+      def add_request_context!(record, env)
+        record.request_context = {
+          ip: request_ip(env),
+          user_agent: env['HTTP_USER_AGENT'],
+          params: filtered_request_params(env).presence,
+          user: current_user_info(env)
+        }.compact
+      rescue StandardError
+        nil
+      end
+
+      def request_ip(env)
+        forwarded = env['HTTP_X_FORWARDED_FOR']
+        forwarded ? forwarded.split(',').first&.strip : env['REMOTE_ADDR']
+      end
+
+      def filtered_request_params(env)
+        params = ActionDispatch::Request.new(env).params.to_h
+        Railswatch::Utils.filter_params(params)
+      rescue StandardError
+        {}
+      end
+
+      def current_user_info(env)
+        return nil unless Railswatch.current_user_proc
+
+        Railswatch.current_user_proc.call(env)&.presence
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/railswatch/rails/query_builder.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Rails
+    class QueryBuilder
+      def self.compose_from(params)
+        result = {}
+
+        result[:controller] = params[:controller_eq]
+        result[:action] = params[:action_eq]
+        result[:format] = params[:format_eq]
+        result[:status] = params[:status_eq]
+
+        result.delete_if { |_k, v| v.nil? }
+
+        { query: result }
+      end
+    end
+  end
+end

data/lib/railswatch/reports/annotations_report.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Reports
+    class AnnotationsReport
+      def data
+        {
+          xaxis: Railswatch::Events::Record.all.map(&:to_annotation)
+        }
+      end
+    end
+  end
+end

data/lib/railswatch/reports/base_report.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Reports
+    class BaseReport
+      attr_reader :db, :group, :sort, :title
+
+      def initialize(db, group: nil, sort: nil, title: nil)
+        @db = db
+        @group = group
+        @sort = sort
+        @title = title
+        set_defaults
+      end
+
+      def set_defaults; end
+
+      def self.time_in_app_time_zone(time)
+        app_time_zone = ::Rails.application.config.time_zone
+        app_time_zone.present? ? time.in_time_zone(app_time_zone) : time
+      end
+
+      def nil_data(duration = Railswatch.duration)
+        @nil_data_cache ||= {}
+        @nil_data_cache[duration] ||= build_nil_data(duration)
+      end
+
+      def nullify_data(input, duration = Railswatch.duration)
+        nil_data(duration).merge(input).sort
+      end
+
+      private
+
+      def build_nil_data(duration)
+        result = {}
+        stop = Railswatch::Utils.kind_of_now.change(sec: 0, usec: 0)
+        current = stop - duration
+
+        while current <= stop
+          result[current.to_i * 1000] = nil
+          current += 1.minute
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/railswatch/reports/breakdown_report.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Reports
+    class BreakdownReport < BaseReport
+      def data
+        db.order(occurred_at: :desc).map(&:record_hash)
+      end
+    end
+  end
+end

data/lib/railswatch/reports/crash_report.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Reports
+    class CrashReport < BaseReport
+      def data
+        db.order(occurred_at: :desc).map(&:record_hash)
+      end
+    end
+  end
+end

data/lib/railswatch/reports/overview_report.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Reports
+    class OverviewReport < BaseReport
+      def data
+        rows = request_rows
+        grouped = grouped_rows(rows)
+        build_overview_data(rows, grouped)
+      end
+
+      private
+
+      def build_overview_data(rows, grouped)
+        {
+          total_requests: rows.size,
+          average_duration: average_duration(rows),
+          error_rate: error_rate(rows),
+          slow_requests: slow_request_count(rows),
+          unique_endpoints: grouped.keys.size,
+          busiest_endpoint: endpoint_volume(grouped),
+          slowest_endpoint: slowest_endpoint(grouped),
+          latest_request_at: latest_request_at(rows)
+        }
+      end
+
+      def request_rows
+        db.pluck(:controller, :action, :status, :duration_ms, :occurred_at)
+      end
+
+      def grouped_rows(rows)
+        rows.group_by { |controller, action, *_rest| "#{controller}##{action}" }
+      end
+
+      def average_duration(rows)
+        return nil if rows.empty?
+
+        rows.sum { |row| duration_value(row) } / rows.size
+      end
+
+      def error_rate(rows)
+        return 0 if rows.empty?
+
+        (error_count(rows).to_f / rows.size) * 100
+      end
+
+      def error_count(rows)
+        rows.count { |row| row[2].to_s.start_with?('5') }
+      end
+
+      def slow_request_count(rows)
+        rows.count { |row| duration_value(row) >= slow_threshold }
+      end
+
+      def slow_threshold
+        Railswatch.slow_requests_threshold.to_f
+      end
+
+      def endpoint_volume(grouped)
+        name, samples = grouped.max_by { |_group_name, group_rows| group_rows.size }
+        { name: name, count: samples&.size.to_i }
+      end
+
+      def slowest_endpoint(grouped)
+        name, samples = grouped.max_by do |_group_name, group_rows|
+          endpoint_p95(group_rows).to_f
+        end
+
+        {
+          name: name,
+          p95_duration: samples.present? ? endpoint_p95(samples) : nil
+        }
+      end
+
+      def endpoint_p95(rows)
+        Railswatch::Utils.percentile(rows.filter_map { |row| row[3] }, 95)
+      end
+
+      def latest_request_at(rows)
+        rows.max_by { |row| row[4].to_i }&.last
+      end
+
+      def duration_value(row)
+        row[3].to_f
+      end
+    end
+  end
+end

data/lib/railswatch/reports/percentile_report.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Reports
+    class PercentileReport < BaseReport
+      def data
+        durations = db.pluck(:duration_ms).compact
+        {
+          p50: Railswatch::Utils.percentile(durations, 50),
+          p95: Railswatch::Utils.percentile(durations, 95),
+          p99: Railswatch::Utils.percentile(durations, 99)
+        }
+      end
+    end
+  end
+end

data/lib/railswatch/reports/recent_requests_report.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Reports
+    class RecentRequestsReport < BaseReport
+      def data
+        time_ago = Railswatch.recent_requests_time_window.ago
+        db.where('occurred_at > ?', time_ago)
+          .order(occurred_at: :desc)
+          .limit(limit)
+          .map(&:record_hash)
+      end
+
+      private
+
+      def limit
+        Railswatch.recent_requests_limit ? Railswatch.recent_requests_limit.to_i : 100_000
+      end
+    end
+  end
+end

data/lib/railswatch/reports/requests_report.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Railswatch
+  module Reports
+    class RequestsReport < BaseReport
+      def set_defaults
+        @set_defaults ||= :count
+      end
+
+      def data
+        rows = grouped_rows.map { |group_name, grouped| build_row(group_name, grouped) }
+        rows.sort_by { |row| -row[sort].to_f }
+      end
+
+      private
+
+      def grouped_rows
+        db.pluck(:controller, :action, :format, :duration_ms, :view_runtime_ms, :db_runtime_ms)
+          .each_with_object(Hash.new { |hash, key| hash[key] = [] }) do |row, grouped|
+            key = group_key(*row.first(3))
+            grouped[key] << metrics_hash(row)
+          end
+      end
+
+      def build_row(group_name, rows) # rubocop:disable Metrics/MethodLength
+        durations = metric_values(rows, :duration_ms)
+        view_runtimes = metric_values(rows, :view_runtime_ms)
+        db_runtimes = metric_values(rows, :db_runtime_ms)
+
+        {
+          group: group_name,
+          count: rows.size,
+          duration_average: average(durations),
+          view_runtime_average: average(view_runtimes),
+          db_runtime_average: average(db_runtimes),
+          duration_slowest: durations.max,
+          view_runtime_slowest: view_runtimes.max,
+          db_runtime_slowest: db_runtimes.max,
+          p50_duration: Railswatch::Utils.percentile(durations, 50),
+          p95_duration: Railswatch::Utils.percentile(durations, 95),
+          p99_duration: Railswatch::Utils.percentile(durations, 99)
+        }
+      end
+
+      def group_key(controller, action, format)
+        case group
+        when :controller_action
+          "#{controller}##{action}"
+        when :controller
+          controller
+        else
+          "#{controller}##{action}|#{format}"
+        end
+      end
+
+      def metrics_hash(row)
+        {
+          duration_ms: row[3],
+          view_runtime_ms: row[4],
+          db_runtime_ms: row[5]
+        }
+      end
+
+      def metric_values(rows, key)
+        rows.filter_map { |row| row[key] }
+      end
+
+      def average(values)
+        return nil if values.empty?
+
+        values.sum.to_f / values.size
+      end
+    end
+  end
+end