rails_performance 1.3.2 → 1.4.0.alpha1

data/config/routes.rb

@@ -14,6 +14,7 @@ RailsPerformance::Engine.routes.draw do
   get "/grape" => "rails_performance#grape", :as => :rails_performance_grape
   get "/rake" => "rails_performance#rake", :as => :rails_performance_rake
   get "/custom" => "rails_performance#custom", :as => :rails_performance_custom
+  get "/resources" => "rails_performance#resources", :as => :rails_performance_resources
 end
 
 Rails.application.routes.draw do

data/lib/generators/rails_performance/install/templates/initializer.rb

@@ -1,7 +1,7 @@
 if defined?(RailsPerformance)
   RailsPerformance.setup do |config|
     # Redis configuration
-    config.redis = Redis::Namespace.new("#{Rails.env}-rails-performance", redis: Redis.new(url: ENV["REDIS_URL"].presence || "redis://127.0.0.1:6379/0"))
+    config.redis = Redis.new(url: ENV["REDIS_URL"].presence || "redis://127.0.0.1:6379/0")
 
     # All data we collect
     config.duration = 4.hours
@@ -36,7 +36,7 @@ if defined?(RailsPerformance)
 
     # You can ignore request paths by specifying the beginning of the path.
     # For example, all routes starting with '/admin' can be ignored:
-    config.ignored_paths = ['/rails/performance']
+    config.ignored_paths = ["/rails/performance"]
 
     # store custom data for the request
     # config.custom_data_proc = proc do |env|

data/lib/rails_performance/data_source.rb

@@ -6,7 +6,8 @@ module RailsPerformance
       delayed_job: RailsPerformance::Models::DelayedJobRecord,
       grape: RailsPerformance::Models::GrapeRecord,
       rake: RailsPerformance::Models::RakeRecord,
-      custom: RailsPerformance::Models::CustomRecord
+      custom: RailsPerformance::Models::CustomRecord,
+      resources: RailsPerformance::Models::ResourceRecord
     }
 
     attr_reader :q, :klass, :type
@@ -20,8 +21,9 @@ module RailsPerformance
 
     def db
       result = RailsPerformance::Models::Collection.new
-      (0..(RailsPerformance::Utils.days + 1)).to_a.reverse_each do |e|
-        RailsPerformance::DataSource.new(q: q.merge({on: (Time.current - e.days).to_date}), type: type).add_to(result)
+      now = Time.current
+      (0..(RailsPerformance::Utils.days)).to_a.reverse_each do |e|
+        RailsPerformance::DataSource.new(q: q.merge({on: (now - e.days).to_date}), type: type).add_to(result)
       end
       result
     end
@@ -53,6 +55,8 @@ module RailsPerformance
       case type
       when :requests
         "performance|*#{compile_requests_query}*|END|#{RailsPerformance::SCHEMA}"
+      when :resources
+        "resource|*#{compile_resource_query}*|END|#{RailsPerformance::SCHEMA}"
       when :sidekiq
         "sidekiq|*#{compile_sidekiq_query}*|END|#{RailsPerformance::SCHEMA}"
       when :delayed_job
@@ -89,6 +93,15 @@ module RailsPerformance
       str.join("*")
     end
 
+    def compile_resource_query
+      str = []
+      str << "server|#{q[:server]}|" if q[:server].present?
+      str << "context|#{q[:context]}|" if q[:context].present?
+      str << "role|#{q[:role]}|" if q[:role].present?
+      str << "datetime|#{q[:on].strftime("%Y%m%d")}*|" if q[:on].present?
+      str.join("*")
+    end
+
     def compile_delayed_job_query
       str = []
       str << "datetime|#{q[:on].strftime("%Y%m%d")}*|" if q[:on].present?

data/lib/rails_performance/engine.rb

@@ -2,11 +2,23 @@ require "action_view/log_subscriber"
 require_relative "rails/middleware"
 require_relative "models/collection"
 require_relative "instrument/metrics_collector"
+require_relative "extensions/resources_monitor"
 
 module RailsPerformance
   class Engine < ::Rails::Engine
     isolate_namespace RailsPerformance
 
+    initializer "rails_performance.resource_monitor" do
+      # check required gems are available
+      RailsPerformance._resource_monitor_enabled = !!(defined?(Sys::Filesystem) && defined?(Sys::CPU) && defined?(GetProcessMem))
+
+      next unless RailsPerformance.enabled
+      next if $rails_performance_running_mode == :console # rubocop:disable Style/GlobalVars
+
+      # start monitoring
+      RailsPerformance._resource_monitor = RailsPerformance::Extensions::ResourceMonitor.new("rails", "web")
+    end
+
     initializer "rails_performance.middleware" do |app|
       next unless RailsPerformance.enabled
 
@@ -24,10 +36,22 @@ module RailsPerformance
 
       if defined?(::Sidekiq)
         require_relative "gems/sidekiq_ext"
+
         Sidekiq.configure_server do |config|
           config.server_middleware do |chain|
             chain.add RailsPerformance::Gems::SidekiqExt
           end
+
+          config.on(:startup) do
+            if $rails_performance_running_mode != :console # rubocop:disable Style/GlobalVars
+              # stop web monitoring
+              # when we run sidekiq it also starts web monitoring (see above)
+              RailsPerformance._resource_monitor.stop_monitoring
+              RailsPerformance._resource_monitor = nil
+              # start background monitoring
+              RailsPerformance._resource_monitor = RailsPerformance::Extensions::ResourceMonitor.new("sidekiq", "background")
+            end
+          end
         end
       end
 
@@ -62,5 +86,9 @@ module RailsPerformance
         RailsPerformance::Gems::RakeExt.init
       end
     end
+
+    if defined?(::Rails::Console)
+      $rails_performance_running_mode = :console # rubocop:disable Style/GlobalVars
+    end
   end
 end

data/lib/rails_performance/extensions/resources_monitor.rb

@@ -0,0 +1,103 @@
+module RailsPerformance
+  module Extensions
+    class ResourceMonitor
+      attr_reader :context, :role
+
+      def initialize(context, role)
+        @context = context
+        @role = role
+        @mutex = Mutex.new
+        @thread = nil
+
+        return unless RailsPerformance._resource_monitor_enabled
+
+        start_monitoring
+      end
+
+      def start_monitoring
+        @mutex.synchronize do
+          return if @thread
+
+          # puts "Starting monitoring for #{context} - #{role}"
+          @thread = Thread.new do
+            loop do
+              run
+            rescue => e
+              ::Rails.logger.error "Monitor error: #{e.message}"
+            ensure
+              sleep 60
+            end
+          end
+        end
+      end
+
+      def stop_monitoring
+        @mutex.synchronize do
+          return unless @thread
+
+          # puts "Stopping monitoring"
+          @thread.kill
+          @thread = nil
+        end
+      end
+
+      def run
+        cpu = fetch_process_cpu_usage
+        memory = fetch_process_memory_usage
+        disk = fetch_disk_usage
+
+        store_data({cpu:, memory:, disk:})
+      end
+
+      def fetch_process_cpu_usage
+        load_averages = Sys::CPU.load_avg
+        {
+          one_min: load_averages[0],
+          five_min: load_averages[1],
+          fifteen_min: load_averages[2]
+        }
+      rescue => e
+        ::Rails.logger.error "Error fetching CPU usage: #{e.message}"
+        {one_min: 0.0, five_min: 0.0, fifteen_min: 0.0}
+      end
+
+      def fetch_process_memory_usage
+        GetProcessMem.new.bytes
+      rescue => e
+        ::Rails.logger.error "Error fetching memory usage: #{e.message}"
+        0
+      end
+
+      def fetch_disk_usage(path = "/")
+        stat = Sys::Filesystem.stat(path)
+        {
+          available: stat.blocks_available * stat.block_size,
+          total: stat.blocks * stat.block_size,
+          used: (stat.blocks - stat.blocks_available) * stat.block_size
+        }
+      rescue => e
+        ::Rails.logger.error "Error fetching disk space: #{e.message}"
+        {available: 0, total: 0, used: 0}
+      end
+
+      def store_data(data)
+        ::Rails.logger.info("Server: #{server_id}, Context: #{context}, Role: #{role}, data: #{data}")
+
+        now = Time.current
+        now = now.change(sec: 0, usec: 0)
+        RailsPerformance::Models::ResourceRecord.new(
+          server: server_id,
+          context: context,
+          role: role,
+          datetime: now.strftime(RailsPerformance::FORMAT),
+          datetimei: now.to_i,
+          json: data
+        ).save
+      end
+
+      def server_id
+        @server_id ||= ENV["SERVER_ID"] || `hostname`.strip
+      end
+    end
+  end
+end

data/lib/rails_performance/models/request_record.rb

@@ -30,7 +30,11 @@ module RailsPerformance
       def self.from_db(key, value)
         items = key.split("|")
 
-        parsed_value = JSON.parse(value) rescue {}
+        parsed_value = begin
+          JSON.parse(value)
+        rescue
+          {}
+        end
 
         RequestRecord.new(
           controller: items[2],
@@ -45,7 +49,7 @@ module RailsPerformance
           json: value,
           duration: parsed_value["duration"],
           view_runtime: parsed_value["view_runtime"],
-          db_runtime: parsed_value["db_runtime"],
+          db_runtime: parsed_value["db_runtime"]
         )
       end
 

data/lib/rails_performance/models/resource_record.rb

@@ -0,0 +1,47 @@
+module RailsPerformance
+  module Models
+    class ResourceRecord < BaseRecord
+      attr_accessor :server, :context, :role, :datetime, :datetimei, :json
+
+      def initialize(server:, context:, role:, datetime:, datetimei:, json:)
+        @server = server
+        @context = context
+        @role = role
+        @datetime = datetime
+        @datetimei = datetimei
+        @json = json
+      end
+
+      def self.from_db(key, value)
+        items = key.split("|")
+
+        ResourceRecord.new(
+          server: items[2],
+          context: items[4],
+          role: items[6],
+          datetime: items[8],
+          datetimei: items[10],
+          json: value
+        )
+      end
+
+      def record_hash
+        {
+          server: server,
+          role: role,
+          context: context,
+          datetime: datetime,
+          datetimei: Time.at(datetimei.to_i),
+          cpu: value["cpu"],
+          memory: value["memory"],
+          disk: value["disk"]
+        }
+      end
+
+      def save
+        key = "resource|server|#{server}|context|#{context}|role|#{role}|datetime|#{datetime}|datetimei|#{datetimei}|END|#{RailsPerformance::SCHEMA}"
+        Utils.save_to_redis(key, json)
+      end
+    end
+  end
+end

data/lib/rails_performance/reports/base_report.rb

@@ -32,6 +32,7 @@ module RailsPerformance
 
       def calculate_data
         now = Time.current
+        now = now.change(sec: 0, usec: 0)
         stop = Time.at(60 * (now.to_i / 60))
         offset = RailsPerformance::Reports::BaseReport.time_in_app_time_zone(now).utc_offset
         current = stop - RailsPerformance.duration
@@ -39,7 +40,6 @@ module RailsPerformance
         @data = []
         all = {}
 
-        # read current values
         db.group_by(group).each do |(k, v)|
           yield(all, k, v)
         end
@@ -48,13 +48,46 @@ module RailsPerformance
         while current <= stop
           key = current.strftime(RailsPerformance::FORMAT)
           views = all[key].presence || 0
-          @data << [(current.to_i + offset) * 1000, views.round(2)]
+          @data << [(current.to_i + offset) * 1000, views.is_a?(Numeric) ? views.round(2) : views]
           current += 1.minute
         end
 
         # sort by time
         @data.sort!
       end
+
+      # Generate series for our time range -> (now - duration)..now
+      # {
+      #   1732125540000 => 0,
+      #   1732125550000 => 0,
+      #   ....
+      # }
+      def nil_data
+        @nil_data ||= begin
+          result = {}
+          now = Time.current
+          now = now.change(sec: 0, usec: 0)
+          stop = Time.at(60 * (now.to_i / 60))
+          offset = RailsPerformance::Reports::BaseReport.time_in_app_time_zone(now).utc_offset
+          current = stop - RailsPerformance.duration
+
+          while current <= stop
+            current.strftime(RailsPerformance::FORMAT)
+            result[(current.to_i + offset) * 1000] = nil
+            current += 1.minute
+          end
+
+          result
+        end
+      end
+
+      # {
+      #   1732125540000 => 1,
+      #   1732125550000 => 0,
+      # }
+      def nullify_data(input)
+        nil_data.merge(input).sort
+      end
     end
   end
 end

data/lib/rails_performance/reports/percentile_report.rb

@@ -2,7 +2,7 @@ module RailsPerformance
   module Reports
     class PercentileReport < BaseReport
       def data
-        durations = db.data.collect(&:duration)
+        durations = db.data.collect(&:duration).compact
         {
           p50: RailsPerformance::Utils.percentile(durations, 50),
           p95: RailsPerformance::Utils.percentile(durations, 95),

data/lib/rails_performance/reports/requests_report.rb

@@ -21,7 +21,7 @@ module RailsPerformance
             db_runtime_slowest: db_runtimes.max,
             p50_duration: RailsPerformance::Utils.percentile(durations, 50),
             p95_duration: RailsPerformance::Utils.percentile(durations, 95),
-            p99_duration: RailsPerformance::Utils.percentile(durations, 99),
+            p99_duration: RailsPerformance::Utils.percentile(durations, 99)
           }
         end.sort_by { |e| -e[sort].to_f } # to_f because could ne NaN or nil
       end

data/lib/rails_performance/reports/resources_report.rb

@@ -0,0 +1,44 @@
+module RailsPerformance
+  module Reports
+    class ResourcesReport < BaseReport
+      def self.x
+        @datasource = RailsPerformance::DataSource.new(type: :resources)
+        db = @datasource.db
+        @data = RailsPerformance::Reports::ResourcesReport.new(db)
+        # RailsPerformance::Reports::ResourcesReport.x
+      end
+
+      def data
+        @data ||= db.data
+          .collect { |e| e.record_hash }
+          .group_by { |e| e[:server] + "///" + e[:context] + "///" + e[:role] }
+          .transform_values { |v| v.sort { |a, b| b[sort] <=> a[sort] } }
+          .transform_values { |v| v.map { |e| e.merge({datetimei: e[:datetimei].to_i}) } }
+      end
+
+      def cpu
+        @cpu ||= data.transform_values do |v|
+          nullify_data(v.each_with_object({}) do |e, res|
+            res[e[:datetimei] * 1000] = e[:cpu]["one_min"].to_f.round(2)
+          end)
+        end
+      end
+
+      def memory
+        @memory ||= data.transform_values do |v|
+          nullify_data(v.each_with_object({}) do |e, res|
+            res[e[:datetimei] * 1000] = e[:memory].to_f.round(2)
+          end)
+        end
+      end
+
+      def disk
+        @disk ||= data.transform_values do |v|
+          nullify_data(v.each_with_object({}) do |e, res|
+            res[e[:datetimei] * 1000] = e[:disk]["available"].to_f.round(2)
+          end)
+        end
+      end
+    end
+  end
+end

data/lib/rails_performance/version.rb

@@ -1,4 +1,4 @@
 module RailsPerformance
-  VERSION = "1.3.2"
+  VERSION = "1.4.0.alpha1"
   SCHEMA = "1.0.1"
 end

data/lib/rails_performance.rb

@@ -11,6 +11,7 @@ require_relative "rails_performance/models/delayed_job_record"
 require_relative "rails_performance/models/grape_record"
 require_relative "rails_performance/models/trace_record"
 require_relative "rails_performance/models/rake_record"
+require_relative "rails_performance/models/resource_record"
 require_relative "rails_performance/models/custom_record"
 require_relative "rails_performance/data_source"
 require_relative "rails_performance/utils"
@@ -24,6 +25,7 @@ require_relative "rails_performance/reports/slow_requests_report"
 require_relative "rails_performance/reports/breakdown_report"
 require_relative "rails_performance/reports/trace_report"
 require_relative "rails_performance/reports/percentile_report"
+require_relative "rails_performance/reports/resources_report"
 require_relative "rails_performance/extensions/trace"
 require_relative "rails_performance/thread/current_request"
 
@@ -117,6 +119,17 @@ module RailsPerformance
   mattr_accessor :ignore_trace_headers
   @@ignore_trace_headers = ["datetimei"]
 
+  mattr_accessor :_resource_monitor
+  @@_resource_monitor = nil
+
+  # to check if we are running in console mode
+  mattr_accessor :_running_mode
+  @@_running_mode = nil
+
+  # by default we don't want to monitor resources, but we can enable it by adding required gems
+  mattr_accessor :_resource_monitor_enabled
+  @@_resource_monitor_enabled = false
+
   def self.setup
     yield(self)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_performance
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.4.0.alpha1
 platform: ruby
 authors:
 - Igor Kasyanchuk
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-08 00:00:00.000000000 Z
+date: 2024-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -248,6 +248,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sys-filesystem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sys-cpu
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: get_process_mem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 3rd party dependency-free solution how to monitor performance of your
   Rails applications.
 email:
@@ -299,6 +341,7 @@ files:
 - app/views/rails_performance/rails_performance/recent.html.erb
 - app/views/rails_performance/rails_performance/recent.js.erb
 - app/views/rails_performance/rails_performance/requests.html.erb
+- app/views/rails_performance/rails_performance/resources.html.erb
 - app/views/rails_performance/rails_performance/sidekiq.html.erb
 - app/views/rails_performance/rails_performance/slow.html.erb
 - app/views/rails_performance/rails_performance/summary.js.erb
@@ -316,6 +359,7 @@ files:
 - lib/rails_performance.rb
 - lib/rails_performance/data_source.rb
 - lib/rails_performance/engine.rb
+- lib/rails_performance/extensions/resources_monitor.rb
 - lib/rails_performance/extensions/trace.rb
 - lib/rails_performance/gems/custom_ext.rb
 - lib/rails_performance/gems/delayed_job_ext.rb
@@ -330,6 +374,7 @@ files:
 - lib/rails_performance/models/grape_record.rb
 - lib/rails_performance/models/rake_record.rb
 - lib/rails_performance/models/request_record.rb
+- lib/rails_performance/models/resource_record.rb
 - lib/rails_performance/models/sidekiq_record.rb
 - lib/rails_performance/models/trace_record.rb
 - lib/rails_performance/rails/middleware.rb
@@ -340,6 +385,7 @@ files:
 - lib/rails_performance/reports/percentile_report.rb
 - lib/rails_performance/reports/recent_requests_report.rb
 - lib/rails_performance/reports/requests_report.rb
+- lib/rails_performance/reports/resources_report.rb
 - lib/rails_performance/reports/response_time_report.rb
 - lib/rails_performance/reports/slow_requests_report.rb
 - lib/rails_performance/reports/throughput_report.rb
@@ -366,7 +412,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Simple Rails Performance tracker. Alternative to the NewRelic, Datadog or