rails-profiler 0.31.4 → 0.31.6

data/lib/profiler/models/profile.rb

@@ -19,7 +19,7 @@ module Profiler
                     :response_body, :response_body_encoding,
                     :request_body_size, :request_body_truncated, :request_body_size_is_minimum,
                     :response_body_size, :response_body_truncated, :response_body_size_is_minimum,
-                    :collectors_released_after_seconds,
+                    :collectors_released_after_seconds, :params_truncated,
                     :gem_version
 
       def initialize(request = nil)
@@ -34,7 +34,8 @@ module Profiler
         if request
           @path = request.path
           @method = request.request_method
-          @params = sanitize_params(request.params)
+          @params, @params_truncated = cap_params(sanitize_params(request.params),
+                                                  Profiler.configuration.max_captured_body_bytes)
           @headers = extract_headers(request.env)
         end
       end
@@ -116,9 +117,15 @@ module Profiler
       end
       private :collector_metadata
 
-      def to_h
-        req_body,  req_enc  = decode_body(@request_body,  @request_body_encoding)
-        resp_body, resp_enc = decode_body(@response_body, @response_body_encoding)
+      # The profile as the stores keep it: a body compressed by compress_bodies stays gzip+base64.
+      # decode_bodies: true gives the bodies as text, for the responses that show them.
+      def to_h(decode_bodies: false)
+        req_body,  req_enc  = @request_body,  @request_body_encoding
+        resp_body, resp_enc = @response_body, @response_body_encoding
+        if decode_bodies
+          req_body,  req_enc  = self.class.decode_body(req_body, req_enc)
+          resp_body, resp_enc = self.class.decode_body(resp_body, resp_enc)
+        end
 
         {
           profile_type: @profile_type,
@@ -133,6 +140,7 @@ module Profiler
           started_at: @started_at&.iso8601,
           finished_at: @finished_at&.iso8601,
           params: @params,
+          params_truncated: @params_truncated,
           headers: @headers,
           response_headers: @response_headers,
           request_body: req_body,
@@ -153,6 +161,7 @@ module Profiler
         }
       end
 
+      # The stored form, as #to_h.
       def to_json(*args)
         to_h.to_json(*args)
       end
@@ -178,6 +187,7 @@ module Profiler
         profile.started_at = data[:started_at] ? Time.parse(data[:started_at]) : nil
         profile.finished_at = data[:finished_at] ? Time.parse(data[:finished_at]) : nil
         profile.params = data[:params]
+        profile.params_truncated = data[:params_truncated]
         profile.headers = data[:headers]
         profile.response_headers = data[:response_headers]
         profile.request_body = data[:request_body]
@@ -209,6 +219,18 @@ module Profiler
         profile
       end
 
+      # A body kept gzip+base64 as text, with its encoding then "text"; any other body as it is.
+      # One that cannot be decompressed (cut, damaged) is returned as it is: unreadable at worst.
+      def self.decode_body(body, encoding)
+        return [body, encoding] unless encoding == "gzip+base64"
+        return [body, encoding] if body.nil? || body.empty?
+
+        decoded = Zlib::Inflate.inflate(Base64.strict_decode64(body))
+        [decoded.force_encoding(Encoding::UTF_8).scrub, "text"]
+      rescue Zlib::Error, ArgumentError
+        [body, encoding]
+      end
+
       def self.deep_stringify_keys(obj)
         case obj
         when Hash
@@ -232,7 +254,8 @@ module Profiler
         else
           text = Redaction.filter_body(raw, content_type).encode("UTF-8", invalid: :replace, undef: :replace)
           if compress_body?(text)
-            { body: Base64.strict_encode64(Zlib::Deflate.deflate(text)), encoding: "gzip+base64" }
+            # The fastest level: paid on the request, a quarter larger than the default level.
+            { body: Base64.strict_encode64(Zlib::Deflate.deflate(text, Zlib::BEST_SPEED)), encoding: "gzip+base64" }
           else
             { body: text, encoding: "text" }
           end
@@ -250,22 +273,67 @@ module Profiler
           text.bytesize > Profiler.configuration.compress_body_threshold
       end
 
-      def decode_body(body, encoding)
-        return [body, encoding] unless encoding == "gzip+base64"
-        return [body, encoding] if body.nil? || body.empty?
-
-        decoded = Zlib::Inflate.inflate(Base64.strict_decode64(body))
-        [decoded, "text"]
-      rescue Zlib::Error, ArgumentError
-        [body, encoding]
-      end
-
       def binary_content_type?(ct)
         ct.to_s.match?(%r{image/(?!svg)|application/(?:pdf|octet-stream|zip)|audio/|video/})
       end
 
       LEGACY_FILTERED_PARAMS = %w[password password_confirmation token secret].freeze
 
+      # The key added to params cut at max_captured_body_bytes, so that the cut shows wherever
+      # the params are read.
+      PARAMS_TRUNCATED_KEY = "[profiler]"
+
+      # A copy of +params+ whose JSON stays within about +limit+ bytes: long strings are cut, and
+      # the entries past the limit left out. The application's own params are never touched.
+      # Returns the params and whether they were cut.
+      def cap_params(params, limit)
+        return [params, false] if limit.nil? || !params.is_a?(Hash)
+
+        budget = [limit]
+        cut = [false]
+        capped = cap_value(params, budget, cut)
+        return [capped, false] unless cut[0]
+
+        capped[PARAMS_TRUNCATED_KEY] = "params truncated at #{limit} bytes (max_captured_body_bytes)"
+        [capped, true]
+      end
+
+      def cap_value(value, budget, cut)
+        case value
+        when Hash
+          value.each_with_object({}) do |(key, item), out|
+            cost = key.to_s.bytesize + 4
+            if cost >= budget[0]
+              cut[0] = true
+              break out
+            end
+            budget[0] -= cost
+            out[key] = cap_value(item, budget, cut)
+          end
+        when Array
+          value.each_with_object([]) do |item, out|
+            if budget[0] <= 1
+              cut[0] = true
+              break out
+            end
+            budget[0] -= 1
+            out << cap_value(item, budget, cut)
+          end
+        when String
+          cost = value.bytesize + 2
+          if cost > budget[0]
+            cut[0] = true
+            value = Redaction.cut_bytes(value, [budget[0] - 2, 0].max).scrub("")
+            cost = value.bytesize + 2
+          end
+          budget[0] -= cost
+          value
+        else
+          budget[0] -= value.to_s.bytesize
+          value
+        end
+      end
+
       def sanitize_params(params)
         return {} unless params
 

data/lib/profiler/process_snapshot.rb

@@ -36,7 +36,11 @@ module Profiler
         candidates.find { |route| route[:verb].split("|").include?(verb) } ||
           candidates.find { |route| route[:verb] == "ANY" } ||
           candidates.first
-      rescue StandardError
+      rescue StandardError => e
+        # A path no route matches (a 404) has no route to show.
+        unless defined?(ActionController::RoutingError) && e.is_a?(ActionController::RoutingError)
+          Profiler.log_error_once(:snapshot_route, "could not find the route of a profile", e)
+        end
         nil
       end
 
@@ -61,7 +65,7 @@ module Profiler
         end
         profile
       rescue StandardError => e
-        warn "Profiler: could not add the routes and ENV to profile #{profile.token}: #{e.message}"
+        Profiler.log_error("could not add the routes and ENV to profile #{profile.token}", e)
         profile
       end
 
@@ -84,7 +88,8 @@ module Profiler
           @routes = build_snapshot(fingerprint, list) unless @routes && @routes[0] == fingerprint
           @routes
         end
-      rescue StandardError
+      rescue StandardError => e
+        Profiler.log_error_once(:snapshot_routes, "could not read the application's routes", e)
         [nil, [].freeze, {}.freeze]
       end
 

data/lib/profiler/railtie.rb

@@ -23,7 +23,7 @@ module Profiler
           if config.respond_to?("#{key}=")
             config.public_send("#{key}=", value)
           else
-            Rails.logger&.warn("[Profiler] config.profiler.#{key} is not a profiler option, ignored")
+            Profiler.log_warn("config.profiler.#{key} is not a profiler option, ignored")
           end
         end
       end

data/lib/profiler/redaction.rb

@@ -518,9 +518,7 @@ module Profiler
         return if reported.include?(error.class)
 
         reported << error.class
-        message = "[Profiler] Redaction: #{error.class} while #{during}; the value was masked instead."
-        logger = ::Rails.logger if defined?(::Rails) && ::Rails.respond_to?(:logger)
-        logger ? logger.warn(message) : Kernel.warn(message)
+        Profiler.log_warn("Redaction: #{error.class} while #{during}; the value was masked instead.")
       rescue StandardError
         nil
       end

data/lib/profiler/sse/redis_event_bus.rb

@@ -29,7 +29,8 @@ module Profiler
       # The version of the last save of +token+, 0 when none is known or Redis cannot be read.
       def version(token)
         redis.get(key(token)).to_i
-      rescue StandardError
+      rescue StandardError => e
+        Profiler.log_error_once(:redis_event_bus_version, "RedisEventBus: could not read the version of a profile", e)
         0
       end
 

data/lib/profiler/storage/base_store.rb

@@ -34,7 +34,8 @@ module Profiler
 
       def exists?(token)
         !load(token).nil?
-      rescue
+      rescue StandardError => e
+        Profiler.log_error_once(:store_exists, "#{self.class.name.split("::").last}: could not look up a profile", e)
         false
       end
 
@@ -54,8 +55,9 @@ module Profiler
 
       def broadcast_event(token)
         Profiler::SSE.current.broadcast(token)
-      rescue StandardError
+      rescue StandardError => e
         # Never let a broadcast failure prevent profile persistence
+        Profiler.log_error_once(:store_broadcast, "#{self.class.name.split("::").last}: could not announce a saved profile", e)
       end
     end
   end

data/lib/profiler/storage/blob_store.rb

@@ -35,7 +35,7 @@ module Profiler
 
         JSON.parse(File.read(file))
       rescue => e
-        warn "BlobStore: failed to read #{token}/#{collector_name}: #{e.message}"
+        Profiler.log_error("BlobStore: could not read #{token}/#{collector_name}", e)
         nil
       end
 

data/lib/profiler/storage/file_store.rb

@@ -100,7 +100,7 @@ module Profiler
         json_data = File.read(file_path)
         Models::Profile.from_json(json_data)
       rescue StandardError => e
-        warn "Failed to load profile #{token}: #{e.message}"
+        Profiler.log_error("FileStore: could not load profile #{token}", e)
         nil
       end
 
@@ -345,9 +345,14 @@ module Profiler
         json = PrivateFiles.open_for_reading(profile_file_path(token), &:read)
         record = put_record(token, Models::Profile.from_json(json).to_h, json.bytesize)
         Entry.new(token, record["at"], record["type"], record["parent"], record["bytes"], record["summary"])
-      rescue StandardError
+      rescue StandardError => e
         # Unreadable: kept in the count, so that it is evicted in turn; listed by no one.
-        stat = File.lstat(profile_file_path(token)) rescue nil
+        Profiler.log_error_once(:file_store_entry, "FileStore: could not read profile #{token} to index it", e)
+        stat = begin
+          File.lstat(profile_file_path(token))
+        rescue SystemCallError
+          nil
+        end
         Entry.new(token, stat ? stat.mtime.to_f : 0.0, nil, nil, stat ? stat.size : 0, nil)
       end
 

data/lib/profiler/storage/private_files.rb

@@ -125,9 +125,9 @@ module Profiler
         first = @warn_mutex.synchronize { @warned[key] ? false : (@warned[key] = true) }
         return unless first
 
-        warn "[Profiler] tmp_path #{key} belongs to another user or is writable by group or others " \
-             "(mode #{format("%o", stat.mode & 0o777)}): other local users could place files there. Set " \
-             "config.tmp_path to a directory of your own, or run chmod 700 on it."
+        Profiler.log_warn("tmp_path #{key} belongs to another user or is writable by group or others " \
+                          "(mode #{format("%o", stat.mode & 0o777)}): other local users could place files there. Set " \
+                          "config.tmp_path to a directory of your own, or run chmod 700 on it.")
       rescue SystemCallError
         nil
       end

data/lib/profiler/storage/redis_store.rb

@@ -64,7 +64,7 @@ module Profiler
 
         Models::Profile.from_json(json_data)
       rescue => e
-        warn "Failed to load profile #{token}: #{e.message}"
+        Profiler.log_error("RedisStore: could not load profile #{token}", e)
         nil
       end
 
@@ -270,7 +270,7 @@ module Profiler
           data = JSON.parse(json, symbolize_names: true)
           [token, data[:started_at] ? Time.parse(data[:started_at]).to_f : 0.0, Summary.build(data)]
         rescue StandardError => e
-          warn "RedisStore: could not index profile #{token}: #{e.message}"
+          Profiler.log_error("RedisStore: could not index profile #{token}", e)
           nil
         end
         @redis.pipelined do |pipe|

data/lib/profiler/storage/sqlite_store.rb

@@ -102,7 +102,7 @@ module Profiler
 
         row_to_profile(row)
       rescue => e
-        warn "SqliteStore: failed to load profile #{token}: #{e.message}"
+        Profiler.log_error("SqliteStore: could not load profile #{token}", e)
         nil
       end
 
@@ -191,7 +191,7 @@ module Profiler
 
         Summary.to_profile(JSON.parse(row["summary"]))
       rescue JSON::ParserError => e
-        warn "SqliteStore: failed to read the summary of #{row["token"]}: #{e.message}"
+        Profiler.log_error("SqliteStore: could not read the summary of #{row["token"]}", e)
         nil
       end
 
@@ -308,7 +308,7 @@ module Profiler
           collectors_data:        collectors_data
         )
       rescue => e
-        warn "SqliteStore: failed to deserialize profile #{row["token"]}: #{e.message}"
+        Profiler.log_error("SqliteStore: could not deserialize profile #{row["token"]}", e)
         nil
       end
 

data/lib/profiler/storage/summary.rb

@@ -7,16 +7,19 @@ module Profiler
     # What the profile lists show of a profile, for the stores to keep apart from the profile
     # itself: the top-level fields without the bodies, params, headers and tabs, and for each
     # collector its scalar values only (database.total_queries, job.queue, test.test_name...),
-    # nested lists and hashes left out, strings cut at MAX_SCALAR_LENGTH (a console return value,
-    # an exception message). Built from Profile#to_h, after the collectors and the
-    # redaction have run: a summary is a subset of the stored profile (a long string cut to its
-    # start) and holds no value the profile does not.
+    # nested lists and hashes and the request collector's copy of the bodies left out, strings
+    # cut at MAX_SCALAR_LENGTH (a console return value, an exception message). Built from
+    # Profile#to_h, after the collectors and the redaction have run: a summary is a subset of the
+    # stored profile (a long string cut to its start) and holds no value the profile does not.
     module Summary
       MAX_SCALAR_LENGTH = 500
 
       DROPPED = %i[params headers response_headers request_body request_body_encoding
                    response_body response_body_encoding tabs].freeze
 
+      # The request collector keeps a copy of the bodies: no more listed than the profile's own.
+      DROPPED_SCALARS = %w[request_body request_body_encoding response_body response_body_encoding].freeze
+
       module_function
 
       # From a Profile, or the Hash its #to_h returns.
@@ -37,6 +40,7 @@ module Profiler
       def scalars(values)
         values.each_with_object({}) do |(key, value), kept|
           next unless scalar?(value)
+          next if DROPPED_SCALARS.include?(key.to_s)
 
           kept[key.to_s] = value.is_a?(String) && value.length > MAX_SCALAR_LENGTH ? value[0, MAX_SCALAR_LENGTH] : value
         end

data/lib/profiler/storage/unavailable.rb

@@ -21,8 +21,7 @@ module Profiler
           key = [Process.pid, error.class.name, error.message]
           first = @mutex.synchronize { @said[key] ? false : (@said[key] = true) }
           if first
-            warn "[Profiler] storage unavailable, profiles are not saved until it is fixed: " \
-                 "#{error.class}: #{error.message}"
+            Profiler.log_warn("storage unavailable, profiles are not saved until it is fixed", error)
           end
           new(error)
         end

data/lib/profiler/test_helpers/reporter.rb

@@ -75,7 +75,7 @@ module Profiler
 
         $stdout.puts lines.join("\n")
       rescue => e
-        warn "Profiler Reporter: failed to generate report: #{e.message}"
+        Profiler.log_error("Reporter: could not generate the report", e)
       end
 
       def self.has_n1?(profile)

data/lib/profiler/test_profiler.rb

@@ -114,7 +114,7 @@ module Profiler
             collector.collect if collector.respond_to?(:collect)
             profile.add_collector_metadata(collector)
           rescue => e
-            warn "Profiler TestProfiler: Collector #{collector.class} failed: #{e.message}"
+            Profiler.log_error("TestProfiler: collector #{collector.class} failed", e)
           end
         end
 

data/lib/profiler/test_runner/runner.rb

@@ -51,7 +51,7 @@ module Profiler
         root = project_root
         allowed = if Profiler.configuration.test_runner_allow_undiscovered_files
                     warn_once(:undiscovered,
-                              "[Profiler] config.test_runner_allow_undiscovered_files is enabled: the test runner " \
+                              "config.test_runner_allow_undiscovered_files is enabled: the test runner " \
                               "runs any file under the Rails root, not only the discovered tests.")
                     nil
                   else
@@ -101,11 +101,7 @@ module Profiler
           @warned[key] = true
         end
 
-        if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
-          Rails.logger.warn(message)
-        else
-          warn(message)
-        end
+        Profiler.log_warn(message)
       end
 
       def self.reset_warnings!
@@ -227,7 +223,7 @@ module Profiler
 
         unless left_out.empty?
           warn_once([:env, left_out.sort],
-                    "[Profiler] Environment overrides not passed to the test runner, because they make it " \
+                    "Environment overrides not passed to the test runner, because they make it " \
                     "load other code: #{left_out.sort.join(", ")}.")
         end
 

data/lib/profiler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Profiler
-  VERSION = "0.31.4"
+  VERSION = "0.31.6"
 end

data/lib/profiler.rb

@@ -10,6 +10,10 @@ require_relative "profiler/allocation_counter"
 module Profiler
   class Error < StandardError; end
 
+  # The longest error message, in bytes, a log line carries: a parser error can quote what it
+  # was parsing.
+  LOG_ERROR_MESSAGE_LIMIT = 1000
+
   class << self
     attr_writer :configuration
     attr_accessor :function_profiling_enabled
@@ -43,11 +47,75 @@ module Profiler
 
     # Saves a profile from a path of the application (a job, a console command, a test, an
     # outbound HTTP call): a storage error loses the profile, never the application's work.
+    # True when saved; false when the error was logged instead, or when the store is unavailable
+    # (Storage::Unavailable drops the save, and has said why once already).
     def save_profile(profile, from:)
-      storage.save(profile.token, profile)
+      target = storage
+      return false if target.is_a?(Storage::Unavailable)
+
+      target.save(profile.token, profile)
+      true
     rescue StandardError => e
-      warn "[Profiler] #{from}: could not save profile #{profile.token}: #{e.class}: #{e.message}"
+      log_error("#{from}: could not save profile #{profile.token}", e)
+      false
+    end
+
+    # The profiler's own messages, to config.logger when the application sets one (read on each
+    # message), else to +logger+ when given (Rails.logger at boot), else to Rails.logger, else to
+    # $stderr.
+    # Prefixed [Profiler], with the profiler's credentials masked. Never raises: a logger that
+    # fails sends the line to $stderr instead. While the line is written, the thread is marked
+    # (Thread.current[:profiler_logging]) so that the Logs tab of a profile being recorded does
+    # not take it for one of the application's lines.
+    def log(level, message, error = nil, backtrace: false, logger: nil)
+      line = log_line(message, error, backtrace)
+      logger = configured_logger || logger || current_logger
+      if logger
+        begin
+          return write_log(logger, level, line)
+        rescue StandardError
+          # Below, to $stderr: the message is not lost.
+        end
+      end
+      $stderr.write("#{line}\n")
       nil
+    rescue StandardError
+      nil
+    end
+
+    def log_error(message, error = nil, backtrace: false)
+      log(:error, message, error, backtrace: backtrace)
+    end
+
+    def log_warn(message, error = nil, logger: nil)
+      log(:warn, message, error, logger: logger)
+    end
+
+    def log_info(message)
+      log(:info, message)
+    end
+
+    # Like log_error, once per +site+ and error class: for the paths called again and again (a
+    # store read on every poll), where the same failure would otherwise fill the log.
+    def log_error_once(site, message, error)
+      key = [site, error.class]
+      @logged_once_mutex.synchronize do
+        return nil if @logged_once.include?(key)
+
+        @logged_once << key
+      end
+      log_error(message, error)
+    end
+
+    # Runs the block without recording its outgoing HTTP calls in the current profile: the
+    # profiler's own calls (the cluster's registration and heartbeats, the master's requests to
+    # its slaves) are not the application's.
+    def untracked_http
+      previous = Thread.current[:profiler_http_untracked]
+      Thread.current[:profiler_http_untracked] = true
+      yield
+    ensure
+      Thread.current[:profiler_http_untracked] = previous
     end
 
     def env_override_store
@@ -94,10 +162,62 @@ module Profiler
       result
     end
 
+    private
+
+    def configured_logger
+      configuration.logger
+    rescue StandardError
+      nil
+    end
+
+    def current_logger
+      configured = configuration.logger
+      return configured if configured
+      return nil unless defined?(::Rails) && ::Rails.respond_to?(:logger)
+
+      ::Rails.logger
+    rescue StandardError
+      nil
+    end
+
+    def write_log(logger, level, line)
+      previous = Thread.current[:profiler_logging]
+      Thread.current[:profiler_logging] = true
+      logger.public_send(level, line)
+    ensure
+      Thread.current[:profiler_logging] = previous
+    end
+
+    # Masked whole, then cut: a cut first could leave the start of a credential, which the
+    # masking by value would no longer find.
+    def log_line(message, error, backtrace)
+      line = +"[Profiler] #{message}"
+      if error
+        line << ": #{error.class}"
+        text = error_text(error)
+        line << ": #{text}" unless text.empty?
+        line << "\n#{error.backtrace.join("\n")}" if backtrace && error.backtrace
+      end
+      Redaction.hide_credentials(line)
+    end
+
+    # The error's message, masked, then cut at LOG_ERROR_MESSAGE_LIMIT bytes. A message that
+    # raises leaves the class alone.
+    def error_text(error)
+      text = Redaction.hide_credentials(error.message.to_s)
+      return text if text.bytesize <= LOG_ERROR_MESSAGE_LIMIT
+
+      "#{Redaction.cut_bytes(text, LOG_ERROR_MESSAGE_LIMIT).scrub("")}..."
+    rescue StandardError
+      ""
+    end
+
+    public
+
     # Dump a variable to the profiler
     # Usage: Profiler.dump(variable, "optional label")
     def dump(value, label = nil)
-      return unless enabled?
+      return value unless enabled?
 
       # The slot exists only while a DumpCollector profiles this thread: outside of one, nobody
       # would ever read the dump, and the thread would keep it for good.
@@ -122,6 +242,9 @@ module Profiler
     end
   end
 
+  @logged_once = Set.new
+  @logged_once_mutex = Mutex.new
+
   self.function_profiling_enabled = true
   self.function_profiling_max_frames = 2000
   self.function_profiling_mode = "lite"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails-profiler
 version: !ruby/object:Gem::Version
-  version: 0.31.4
+  version: 0.31.6
 platform: ruby
 authors:
 - Sébastien Duplessy