rails-profiler 0.31.0 → 0.31.2

data/lib/profiler/models/profile.rb

@@ -5,17 +5,21 @@ require "securerandom"
 require "json"
 require "zlib"
 require_relative "../redaction"
+require_relative "../allocation_counter"
 
 module Profiler
   module Models
     class Profile
       attr_reader :path
-      attr_accessor :token, :method, :status, :duration, :memory,
+      attr_accessor :token, :method, :status, :duration, :allocated_objects,
                     :started_at, :finished_at, :params, :headers,
                     :response_headers, :collectors_data, :collectors_metadata,
                     :parent_token, :is_ajax, :profile_type,
                     :request_body, :request_body_encoding,
                     :response_body, :response_body_encoding,
+                    :request_body_size, :request_body_truncated, :request_body_size_is_minimum,
+                    :response_body_size, :response_body_truncated, :response_body_size_is_minimum,
+                    :collectors_released_after_seconds,
                     :gem_version
 
       def initialize(request = nil)
@@ -35,13 +39,32 @@ module Profiler
         end
       end
 
-      def set_bodies(request_body:, response_body:, req_content_type:, resp_content_type:)
+      # The sizes are those of the whole bodies, when a body was cut at max_captured_body_bytes;
+      # *_size_is_minimum when the whole size is not known (no Content-Length, a stream that
+      # stopped), only that it is at least that.
+      def set_bodies(request_body:, response_body:, req_content_type:, resp_content_type:,
+                     request_body_size: nil, response_body_size: nil,
+                     request_body_size_is_minimum: false, response_body_size_is_minimum: false)
         req  = process_body(request_body, req_content_type)
         resp = process_body(response_body, resp_content_type)
         @request_body          = req[:body]
         @request_body_encoding = req[:encoding]
         @response_body         = resp[:body]
         @response_body_encoding = resp[:encoding]
+        @request_body_size, @request_body_truncated = body_size(request_body, request_body_size)
+        @response_body_size, @response_body_truncated = body_size(response_body, response_body_size)
+        @request_body_size_is_minimum = request_body_size_is_minimum ? true : false
+        @response_body_size_is_minimum = response_body_size_is_minimum ? true : false
+      end
+
+      # Deprecated: the number of allocated objects times 40, the figure earlier versions
+      # reported as bytes. Still written in to_h, for the readers that only know it.
+      def memory
+        @allocated_objects&.*(AllocationCounter::LEGACY_BYTES_PER_OBJECT)
+      end
+
+      def memory=(value)
+        @allocated_objects = value && value / AllocationCounter::LEGACY_BYTES_PER_OBJECT
       end
 
       def finish(status, response_headers = {})
@@ -67,8 +90,20 @@ module Profiler
       end
 
       def add_collector_metadata(collector)
+        @collectors_metadata << collector_metadata(collector)
+      end
+
+      # A collector collected again, after the profile's tabs were listed (a streamed response
+      # that failed part way): its tab keeps its place, with what it now has.
+      def refresh_collector_metadata(collector)
+        entry = collector_metadata(collector)
+        index = @collectors_metadata.index { |tab| tab[:key] == entry[:key] }
+        index ? @collectors_metadata[index] = entry : @collectors_metadata << entry
+      end
+
+      def collector_metadata(collector)
         config = collector.tab_config
-        @collectors_metadata << {
+        {
           key: config[:key],
           label: config[:label],
           icon: config[:icon],
@@ -79,6 +114,7 @@ module Profiler
           has_data: collector.has_data?
         }
       end
+      private :collector_metadata
 
       def to_h
         req_body,  req_enc  = decode_body(@request_body,  @request_body_encoding)
@@ -92,7 +128,8 @@ module Profiler
           method: @method,
           status: @status,
           duration: @duration,
-          memory: @memory,
+          allocated_objects: @allocated_objects,
+          memory: memory,
           started_at: @started_at&.iso8601,
           finished_at: @finished_at&.iso8601,
           params: @params,
@@ -102,6 +139,13 @@ module Profiler
           request_body_encoding: req_enc,
           response_body: resp_body,
           response_body_encoding: resp_enc,
+          request_body_size: @request_body_size,
+          request_body_truncated: @request_body_truncated,
+          request_body_size_is_minimum: @request_body_size_is_minimum,
+          response_body_size: @response_body_size,
+          response_body_truncated: @response_body_truncated,
+          response_body_size_is_minimum: @response_body_size_is_minimum,
+          collectors_released_after_seconds: @collectors_released_after_seconds,
           collectors_data: @collectors_data,
           tabs: @collectors_metadata,
           parent_token: @parent_token,
@@ -125,7 +169,12 @@ module Profiler
         profile.method = data[:method]
         profile.status = data[:status]
         profile.duration = data[:duration]
-        profile.memory = data[:memory]
+        # A profile saved before allocated_objects only carries memory, the count times 40.
+        if data.key?(:allocated_objects)
+          profile.allocated_objects = data[:allocated_objects]
+        else
+          profile.memory = data[:memory]
+        end
         profile.started_at = data[:started_at] ? Time.parse(data[:started_at]) : nil
         profile.finished_at = data[:finished_at] ? Time.parse(data[:finished_at]) : nil
         profile.params = data[:params]
@@ -135,6 +184,15 @@ module Profiler
         profile.request_body_encoding = data[:request_body_encoding] || "text"
         profile.response_body = data[:response_body]
         profile.response_body_encoding = data[:response_body_encoding] || "text"
+        profile.request_body_size = data[:request_body_size]
+        # Left nil when the store did not keep them (SqliteStore rebuilds a profile from its
+        # columns): the request collector's data still has them.
+        profile.request_body_truncated = data[:request_body_truncated]
+        profile.request_body_size_is_minimum = data[:request_body_size_is_minimum]
+        profile.response_body_size = data[:response_body_size]
+        profile.response_body_truncated = data[:response_body_truncated]
+        profile.response_body_size_is_minimum = data[:response_body_size_is_minimum]
+        profile.collectors_released_after_seconds = data[:collectors_released_after_seconds]
         profile.parent_token = data[:parent_token]
         profile.is_ajax = data[:is_ajax] || false
         profile.profile_type = data[:profile_type] || "http"
@@ -181,6 +239,12 @@ module Profiler
         end
       end
 
+      def body_size(captured, total)
+        captured_size = captured.to_s.bytesize
+        total ||= captured_size
+        [total, total > captured_size]
+      end
+
       def compress_body?(text)
         Profiler.configuration.compress_bodies &&
           text.bytesize > Profiler.configuration.compress_body_threshold

data/lib/profiler/redaction.rb

@@ -88,6 +88,16 @@ module Profiler
         text.length > max ? "#{text[0, max]}#{ellipsis}" : text
       end
 
+      # The first +max+ bytes of +text+, a body kept only in part. A credential cut in two by the
+      # limit would escape the masking by value, which runs later on what is kept: the bytes
+      # that could start one are left out as well. Text no longer than +max+ is returned whole.
+      def cut_bytes(text, max)
+        return text if text.bytesize <= max
+
+        cut = text.byteslice(0, max)
+        cut.byteslice(0, cut.bytesize - held_back_bytes(cut))
+      end
+
       # How many trailing bytes of +text+, a stream read in pieces, could be
       # the start of a credential and have to wait for the next piece: the
       # longest end of the text that is also the beginning of a credential,

data/lib/profiler/storage/base_store.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
+require_relative "token"
+
 module Profiler
   module Storage
     class BaseStore
       # Public interface: persists the profile then records the save, for the pages showing it
       # to see it changed (Profiler::SSE).
-      # Subclasses implement #do_save, not #save.
+      # Subclasses implement #do_save, not #save. The token must be one the gem issued
+      # (Storage::Token); every other public method answers "not found" for any other token.
       def save(token, profile)
+        Token.validate!(token)
         result = do_save(token, profile)
         broadcast_event(token)
         result

data/lib/profiler/storage/blob_store.rb

@@ -2,22 +2,34 @@
 
 require "fileutils"
 require "json"
+require_relative "private_files"
+require_relative "token"
 
 module Profiler
   module Storage
+    # Large collector payloads of the SQLite store, one directory per profile token and one JSON
+    # file per collector. A token or a name that is not one the gem writes is never a path: reads
+    # find nothing and writes are refused.
     class BlobStore
+      NAME_FORMAT = /\A[a-z0-9_]+\z/
+
       def initialize(path)
-        @path = path
-        FileUtils.mkdir_p(@path)
+        @path = path.to_s
+        PrivateFiles.mkdir(@path)
       end
 
       def write(token, collector_name, data)
+        Token.validate!(token)
+        raise ArgumentError, "invalid blob name: #{collector_name.inspect[0, 80]}" unless valid_name?(collector_name)
+
         dir = token_dir(token)
-        FileUtils.mkdir_p(dir)
-        File.write(File.join(dir, "#{collector_name}.json"), JSON.generate(data))
+        PrivateFiles.mkdir(dir)
+        PrivateFiles.write(File.join(dir, "#{collector_name}.json"), JSON.generate(data))
       end
 
       def read(token, collector_name)
+        return nil unless valid?(token, collector_name)
+
         file = File.join(token_dir(token), "#{collector_name}.json")
         return nil unless File.exist?(file)
 
@@ -28,16 +40,28 @@ module Profiler
       end
 
       def delete(token)
+        return unless Token.valid?(token)
+
         dir = token_dir(token)
         FileUtils.rm_rf(dir) if File.directory?(dir)
       end
 
       def exists?(token, collector_name)
+        return false unless valid?(token, collector_name)
+
         File.exist?(File.join(token_dir(token), "#{collector_name}.json"))
       end
 
       private
 
+      def valid?(token, collector_name)
+        Token.valid?(token) && valid_name?(collector_name)
+      end
+
+      def valid_name?(collector_name)
+        collector_name.is_a?(String) && NAME_FORMAT.match?(collector_name)
+      end
+
       def token_dir(token)
         File.join(@path, token)
       end

data/lib/profiler/storage/file_store.rb

@@ -3,28 +3,38 @@
 require "fileutils"
 require "json"
 require_relative "base_store"
+require_relative "private_files"
+require_relative "token"
 require_relative "../models/profile"
 
 module Profiler
   module Storage
-    # File-based profile storage backend. Persists each profile as a JSON file
-    # under tmp_path and evicts the oldest files when total size exceeds max_size.
+    # File-based profile storage backend. Persists each profile as a JSON file named after its
+    # token, under tmp_path/profiles by default or the directory given as options[:path], and
+    # evicts the oldest files when total size exceeds max_size. Only the files named after a
+    # token are taken for profiles, so a path shared with other files (tmp_path itself, where the
+    # env overrides live) is safe.
     class FileStore < BaseStore
       def initialize(options = {})
         super()
-        @path = options[:path] || default_path
         @max_size = options[:max_size] || (100 * 1024 * 1024) # 100 MB
-        ensure_directory_exists
+        if options[:path]
+          @path = options[:path].to_s
+          PrivateFiles.mkdir(@path)
+        else
+          @path = PrivateFiles.tmp_dir("profiles").to_s
+        end
       end
 
       def do_save(token, profile)
-        file_path = profile_file_path(token)
-        File.write(file_path, profile.to_json)
+        PrivateFiles.write(profile_file_path(token), profile.to_json)
         cleanup_if_needed
         token
       end
 
       def load(token)
+        return nil unless Token.valid?(token)
+
         file_path = profile_file_path(token)
         return nil unless File.exist?(file_path)
 
@@ -55,6 +65,8 @@ module Profiler
       end
 
       def find_by_parent(parent_token)
+        return [] unless Token.valid?(parent_token)
+
         profile_files
           .map { |f| load(File.basename(f, ".json")) }
           .compact
@@ -63,6 +75,8 @@ module Profiler
       end
 
       def delete(token)
+        return unless Token.valid?(token)
+
         FileUtils.rm_f(profile_file_path(token))
       end
 
@@ -85,20 +99,12 @@ module Profiler
 
       private
 
-      def default_path
-        Profiler.configuration.tmp_path
-      end
-
-      def ensure_directory_exists
-        FileUtils.mkdir_p(@path) unless File.directory?(@path)
-      end
-
       def profile_file_path(token)
         File.join(@path, "#{token}.json")
       end
 
       def profile_files
-        Dir.glob(File.join(@path, "*.json"))
+        Dir.glob(File.join(@path, "*.json")).select { |f| Token.valid?(File.basename(f, ".json")) }
       end
 
       def cleanup_if_needed # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

data/lib/profiler/storage/memory_store.rb

@@ -19,6 +19,8 @@ module Profiler
       end
 
       def load(token)
+        return nil unless Token.valid?(token)
+
         data = @profiles[token]
         return nil unless data
 
@@ -43,6 +45,8 @@ module Profiler
       end
 
       def find_by_parent(parent_token)
+        return [] unless Token.valid?(parent_token)
+
         @profiles.values
                 .map { |data| deserialize_profile(data) }
                 .select { |profile| profile.parent_token == parent_token }
@@ -50,6 +54,8 @@ module Profiler
       end
 
       def delete(token)
+        return unless Token.valid?(token)
+
         @profiles.delete(token)
       end
 

data/lib/profiler/storage/private_files.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "securerandom"
+
+module Profiler
+  module Storage
+    # Creates the profiler's directories and files readable by the process user only (0700 and
+    # 0600), whatever the umask: profiles hold cookies, tokens and environment values. A directory
+    # that already exists keeps its mode. config.restrict_storage_permissions = false leaves the
+    # modes to the umask, for a web process and workers running under two users.
+    module PrivateFiles
+      DIR_MODE = 0o700
+      FILE_MODE = 0o600
+      # Where the platform has it, and while the modes are restricted: a symbolic link placed where
+      # the profiler expects its own file is not followed (ELOOP) instead of being written through.
+      NOFOLLOW = defined?(File::NOFOLLOW) ? File::NOFOLLOW : 0
+
+      @warned = {}
+      @warn_mutex = Mutex.new
+
+      module_function
+
+      def restricted?
+        Profiler.configuration.restrict_storage_permissions ? true : false
+      end
+
+      # Creates the missing parents with the umask's mode, so that the application's own
+      # directories (tmp/ for instance) are left as they would be, and the directory itself 0700.
+      def mkdir(path)
+        path = path.to_s
+        return if File.directory?(path)
+        return FileUtils.mkdir_p(path) unless restricted?
+
+        FileUtils.mkdir_p(File.dirname(path))
+        begin
+          Dir.mkdir(path, DIR_MODE)
+          File.chmod(DIR_MODE, path)
+        rescue Errno::EEXIST
+          raise unless File.directory?(path)
+        end
+      end
+
+      # A directory under tmp_path, created with tmp_path itself as #mkdir creates a directory.
+      def tmp_dir(*parts)
+        dir = Profiler.configuration.tmp_path
+        mkdir(dir)
+        warn_if_shared(dir)
+        parts.each do |part|
+          dir = dir.join(part)
+          mkdir(dir)
+        end
+        dir
+      end
+
+      # Writes to a temporary file next to path then renames it over path: a reader never sees a
+      # half-written file, and the file has its mode from the start, even when it replaces an
+      # older one with a wider mode.
+      def write(path, data)
+        path = path.to_s
+        tmp = File.join(File.dirname(path), ".#{File.basename(path)}.#{Process.pid}.#{SecureRandom.hex(4)}.tmp")
+        File.open(tmp, File::WRONLY | File::CREAT | File::EXCL, new_file_mode) do |file|
+          file.chmod(FILE_MODE) if restricted?
+          file.write(data)
+        end
+        File.rename(tmp, path)
+        tmp = nil
+        path
+      ensure
+        FileUtils.rm_f(tmp) if tmp
+      end
+
+      # Opens path for reading and writing, creating it 0600 when missing.
+      def open(path, &block)
+        File.open(path.to_s, File::RDWR | File::CREAT | nofollow, new_file_mode, &block)
+      end
+
+      # Creates an empty file 0600 when missing, or brings an existing one back to 0600.
+      def touch(path)
+        path = path.to_s
+        refuse_link(path)
+        File.open(path, File::WRONLY | File::CREAT | nofollow, new_file_mode) { |file| file.chmod(FILE_MODE) if restricted? }
+      end
+
+      # Brings an existing file back to 0600; a missing one is left missing.
+      def restrict(path)
+        path = path.to_s
+        refuse_link(path)
+        File.chmod(FILE_MODE, path) if restricted? && File.exist?(path)
+      end
+
+      # A symbolic link where the profiler keeps a file of its own would make it change the mode of
+      # (and write to) whatever the link points at. Refused while the permissions are restricted.
+      def refuse_link(path)
+        return unless restricted? && File.symlink?(path)
+
+        raise Profiler::Error, "Refusing to use #{path}: it is a symbolic link, where the profiler expects " \
+                               "a file of its own. Remove it, or set config.restrict_storage_permissions = false, " \
+                               "which follows such links and creates the files with the modes of the umask."
+      end
+
+      # Says once per directory when an existing tmp_path belongs to another user or can be written
+      # to by others (tmp/rails-profiler under a shared /tmp, outside Rails): they could place files
+      # or links there before the profiler does.
+      def warn_if_shared(dir)
+        return unless restricted?
+
+        stat = File.stat(dir.to_s)
+        return if stat.uid == Process.euid && (stat.mode & 0o022).zero?
+
+        key = dir.to_s
+        first = @warn_mutex.synchronize { @warned[key] ? false : (@warned[key] = true) }
+        return unless first
+
+        warn "[Profiler] tmp_path #{key} belongs to another user or is writable by group or others " \
+             "(mode #{format("%o", stat.mode & 0o777)}): other local users could place files there. Set " \
+             "config.tmp_path to a directory of your own, or run chmod 700 on it."
+      rescue SystemCallError
+        nil
+      end
+
+      def reset_warnings!
+        @warn_mutex.synchronize { @warned = {} }
+      end
+
+      def nofollow
+        restricted? ? NOFOLLOW : 0
+      end
+
+      def new_file_mode
+        restricted? ? FILE_MODE : 0o666
+      end
+    end
+  end
+end

data/lib/profiler/storage/redis_store.rb

@@ -29,6 +29,8 @@ module Profiler
       end
 
       def load(token)
+        return nil unless Token.valid?(token)
+
         key = profile_key(token)
         json_data = @redis.get(key)
         return nil unless json_data
@@ -50,6 +52,8 @@ module Profiler
       end
 
       def find_by_parent(parent_token)
+        return [] unless Token.valid?(parent_token)
+
         # Get all tokens from the sorted set
         tokens = @redis.zrange(list_key, 0, -1)
 
@@ -61,6 +65,8 @@ module Profiler
       end
 
       def delete(token)
+        return unless Token.valid?(token)
+
         @redis.del(profile_key(token))
         @redis.zrem(list_key, token)
       end

data/lib/profiler/storage/sqlite_store.rb

@@ -4,6 +4,8 @@ require "json"
 require "fileutils"
 require_relative "base_store"
 require_relative "blob_store"
+require_relative "private_files"
+require_relative "token"
 require_relative "../models/profile"
 
 module Profiler
@@ -12,12 +14,12 @@ module Profiler
       def initialize(options = {})
         require "sqlite3"
 
-        db_path = options[:database] || default_db_path
-        blob_path = options[:blob_path] || default_blob_path
+        db_path = (options[:database] || default_db_path).to_s
+        blob_path = options[:blob_path] || PrivateFiles.tmp_dir("blobs")
 
-        FileUtils.mkdir_p(File.dirname(db_path))
+        prepare_database_file(db_path, default: options[:database].nil?)
 
-        @db = SQLite3::Database.new(db_path.to_s)
+        @db = SQLite3::Database.new(db_path)
         @db.results_as_hash = true
         @db.busy_timeout = 5000
         @db.execute("PRAGMA journal_mode=WAL")
@@ -83,6 +85,8 @@ module Profiler
       end
 
       def load(token)
+        return nil unless Token.valid?(token)
+
         row = @db.get_first_row(
           "SELECT * FROM profiler_profiles WHERE token = :token", token: token
         )
@@ -103,6 +107,8 @@ module Profiler
       end
 
       def find_by_parent(parent_token)
+        return [] unless Token.valid?(parent_token)
+
         rows = @db.execute(
           "SELECT * FROM profiler_profiles WHERE parent_token = :parent_token ORDER BY started_at ASC",
           parent_token: parent_token
@@ -111,6 +117,8 @@ module Profiler
       end
 
       def delete(token)
+        return unless Token.valid?(token)
+
         @db.execute("DELETE FROM profiler_profiles WHERE token = :token", token: token)
         @blob_store.delete(token)
       end
@@ -264,13 +272,23 @@ module Profiler
         default
       end
 
-      def default_db_path
-        File.join(Profiler.configuration.tmp_path.to_s, "profiler.db")
+      # SQLite gives the -wal and -shm files the mode of the database file: create it 0600 before
+      # SQLite does, and bring an older one, and its companions, back to 0600. An in-memory or URI
+      # database is left to SQLite.
+      def prepare_database_file(db_path, default:)
+        return if db_path == ":memory:" || db_path.start_with?("file:")
+
+        default ? PrivateFiles.tmp_dir : PrivateFiles.mkdir(File.dirname(db_path))
+        # A link in place of one of them is refused: SQLite would follow it.
+        %w[-wal -shm].each { |suffix| PrivateFiles.refuse_link("#{db_path}#{suffix}") }
+        PrivateFiles.touch(db_path)
+        %w[-wal -shm].each { |suffix| PrivateFiles.restrict("#{db_path}#{suffix}") }
       end
 
-      def default_blob_path
-        File.join(Profiler.configuration.tmp_path.to_s, "blobs")
+      def default_db_path
+        Profiler.configuration.tmp_path.join("profiler.db")
       end
+
     end
   end
 end

data/lib/profiler/storage/token.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Profiler
+  module Storage
+    # The format of a profile token, as Models::Profile issues it (SecureRandom.hex(16)) for every
+    # kind of profile. The backends check a token against it before they turn it into a file name,
+    # a directory or a key: a token is client input for the controllers, the cluster proxy and the
+    # MCP tools, which all call the storage directly.
+    module Token
+      FORMAT = /\A[0-9a-f]{32}\z/
+
+      def self.valid?(token)
+        token.is_a?(String) && FORMAT.match?(token)
+      end
+
+      # For the writers: the gem only writes under the tokens it issued, so another one is a bug.
+      def self.validate!(token)
+        raise ArgumentError, "invalid profile token: #{token.inspect[0, 80]}" unless valid?(token)
+
+        token
+      end
+    end
+  end
+end

data/lib/profiler/test_profiler.rb

@@ -56,7 +56,7 @@ module Profiler
 
       exception_collector = collectors.find { |c| c.is_a?(Collectors::ExceptionCollector) }
 
-      memory_before = current_memory if Profiler.configuration.track_memory
+      allocations_before = AllocationCounter.current if Profiler.configuration.track_memory
 
       test_status = "passed"
       error_message = nil
@@ -103,7 +103,7 @@ module Profiler
         Profiler::CurrentContext.token = previous_token
 
         if Profiler.configuration.track_memory
-          profile.memory = current_memory - memory_before
+          profile.allocated_objects = AllocationCounter.current - allocations_before
         end
 
         test_collector.update_status(test_status, error_message)
@@ -118,26 +118,11 @@ module Profiler
           end
         end
 
-        Profiler.storage.save(profile.token, profile)
+        Profiler.save_profile(profile, from: "TestProfiler")
       end
     ensure
       # After collect, and also when collect or the storage failed.
       Collectors::Lifecycle.release_all(collectors)
     end
-
-    private
-
-    def current_memory
-      return 0 unless defined?(GC.stat)
-
-      stats = GC.stat
-      if stats.key?(:total_allocated_size)
-        stats[:total_allocated_size]
-      elsif stats.key?(:total_allocated_objects)
-        stats[:total_allocated_objects] * 40
-      else
-        0
-      end
-    end
   end
 end

data/lib/profiler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Profiler
-  VERSION = "0.31.0"
+  VERSION = "0.31.2"
 end