rails-profiler 0.31.2 → 0.31.4

data/lib/profiler/mcp/tools/run_tests.rb

@@ -114,7 +114,7 @@ module Profiler
         end
 
         def self.collect_run_profiles(since)
-          Profiler.storage.list(limit: 500).select do |p|
+          Profiler.storage.list(limit: 500, type: "test").select do |p|
             p.profile_type == "test" && p.started_at && p.started_at >= since
           end.map(&:token)
         rescue

data/lib/profiler/middleware/profiler_middleware.rb

@@ -104,7 +104,10 @@ module Profiler
         record_response(env, profile, request_body, status, headers, content, content.bytesize, true)
         collect_all(profile, collectors)
 
-        Profiler.storage.save(profile.token, profile)
+        storage = Profiler.storage
+        storage.save(profile.token, profile)
+        # Not saved: no token to send, no toolbar to inject for it.
+        return [status, headers, body] if storage.is_a?(Profiler::Storage::Unavailable)
 
         set_header(headers, TOKEN_HEADER, profile.token)
 

data/lib/profiler/process_snapshot.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require_relative "redaction"
+
+module Profiler
+  # What is the same for every request of a process, kept out of the profiles: the route table
+  # and ENV. A profile stores the route its request matched; the Routes and Env tabs, the
+  # toolbar and the MCP tools get the table and the variables from the process that displays the
+  # profile (#hydrate).
+  #
+  # The route table is built once and built again when the routes are reloaded, in development
+  # (reload_routes!, or a change picked up by the reloader): the routes are new objects then.
+  # ENV is read when the profile is displayed, masked as it always was: it can change while the
+  # process runs, through the profiler's own env overrides among others.
+  module ProcessSnapshot
+    @mutex = Mutex.new
+    @routes = nil # [fingerprint, table, routes by [controller, action]]
+
+    class << self
+      # The application's routes, the engine's and Rails' own left out, as the Routes tab lists
+      # them: { name:, pattern:, verb:, controller_action: }.
+      def route_table
+        routes_snapshot[1]
+      end
+
+      # The route of the table a request to +path+ with +method+ is dispatched to, or nil.
+      def matched_route(path, method)
+        return nil unless rails_routes?
+
+        recognized = Rails.application.routes.recognize_path(path, method: method)
+        candidates = routes_snapshot[2][[recognized[:controller], recognized[:action]]]
+        return nil unless candidates
+
+        # A PUT goes to the action of the PATCH route too: give the route of the request's verb.
+        verb = method.to_s.upcase == "HEAD" ? "GET" : method.to_s.upcase
+        candidates.find { |route| route[:verb].split("|").include?(verb) } ||
+          candidates.find { |route| route[:verb] == "ANY" } ||
+          candidates.first
+      rescue StandardError
+        nil
+      end
+
+      # ENV as the Env tab shows it.
+      def env_variables
+        Redaction.hide_credentials(Redaction.env_snapshot)
+      end
+
+      # Puts the route table and ENV back into +profile+'s data for display. A profile saved by
+      # an earlier version has its own table and variables, and is left as it is.
+      def hydrate(profile)
+        # A job, a console expression or a test ran in another process, whose ENV its profile keeps.
+        return profile unless profile && profile.profile_type.to_s == "http"
+
+        data = profile.collectors_data
+        if (routes = data["routes"]) && !key?(routes, :routes)
+          data["routes"] = with_route_table(routes)
+        end
+        if (env = data["env"]) && !key?(env, :variables)
+          variables = env_variables
+          data["env"] = merge(env, variables: variables, total: variables.size)
+        end
+        profile
+      rescue StandardError => e
+        warn "Profiler: could not add the routes and ENV to profile #{profile.token}: #{e.message}"
+        profile
+      end
+
+      private
+
+      def rails_routes?
+        defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+      end
+
+      def routes_snapshot
+        return [nil, [].freeze, {}.freeze] unless rails_routes?
+
+        journey = Rails.application.routes.routes
+        list = journey.respond_to?(:routes) ? journey.routes : journey.to_a
+        fingerprint = [journey.object_id, list.size, list.first&.object_id, list.last&.object_id]
+        snapshot = @routes
+        return snapshot if snapshot && snapshot[0] == fingerprint
+
+        @mutex.synchronize do
+          @routes = build_snapshot(fingerprint, list) unless @routes && @routes[0] == fingerprint
+          @routes
+        end
+      rescue StandardError
+        [nil, [].freeze, {}.freeze]
+      end
+
+      def build_snapshot(fingerprint, list)
+        index = {}
+        table = list.filter_map do |route|
+          next if route.respond_to?(:internal) && route.internal
+
+          controller = route.defaults[:controller]
+          action = route.defaults[:action]
+          next if controller.nil?
+          next if controller.start_with?("rails/", "profiler/")
+
+          verb = route.verb
+          entry = {
+            name: route.name,
+            pattern: route.path.spec.to_s.sub(/\(\.:format\)$/, ""),
+            verb: (verb && !verb.empty?) ? verb : "ANY",
+            controller_action: controller_action(controller, action)
+          }.freeze
+          (index[[controller, action]] ||= []) << entry
+          entry
+        end
+        [fingerprint, table.freeze, index.each_value(&:freeze).freeze]
+      end
+
+      def controller_action(controller, action)
+        return nil unless action
+
+        "#{controller.split("/").map { |s| ActiveSupport::Inflector.camelize(s) }.join("::")}Controller##{action}"
+      end
+
+      def with_route_table(routes)
+        matched = fetch(routes, :matched)
+        table = route_table.map { |route| route.merge(matched: same_route?(route, matched)) }
+        merge(routes, routes: table, total: table.size)
+      end
+
+      # Every route of the matched action, as the table always marked them.
+      def same_route?(route, matched)
+        return false unless matched.is_a?(Hash)
+
+        action = fetch(matched, :controller_action)
+        action ? route[:controller_action] == action : route[:pattern] == fetch(matched, :pattern)
+      end
+
+      # Collector data has symbol keys as collected, string keys once read back from storage.
+      def key?(hash, key)
+        hash.key?(key) || hash.key?(key.to_s)
+      end
+
+      def fetch(hash, key)
+        hash.key?(key) ? hash[key] : hash[key.to_s]
+      end
+
+      def merge(hash, values)
+        strings = hash.keys.first.is_a?(String)
+        hash.merge(strings ? deep_stringify(values) : values)
+      end
+
+      def deep_stringify(value)
+        case value
+        when Hash then value.to_h { |k, v| [k.to_s, deep_stringify(v)] }
+        when Array then value.map { |v| deep_stringify(v) }
+        else value
+        end
+      end
+    end
+  end
+end

data/lib/profiler/railtie.rb

@@ -15,6 +15,9 @@ module Profiler
         config.default(:enabled, development_or_test)
         config.default(:storage, development_or_test ? :file : :memory)
         config.default(:track_tests, Rails.env.test?)
+        # One test profile per example: no cap on the count in the test environment, so that the
+        # report at the end of a suite sees all of them.
+        config.default(:max_profiles, Rails.env.test? ? nil : 100)
 
         app.config.profiler.each do |key, value|
           if config.respond_to?("#{key}=")

data/lib/profiler/storage/file_store.rb

@@ -2,33 +2,92 @@
 
 require "fileutils"
 require "json"
+require "securerandom"
+require "time"
 require_relative "base_store"
 require_relative "private_files"
+require_relative "summary"
 require_relative "token"
 require_relative "../models/profile"
 
 module Profiler
   module Storage
     # File-based profile storage backend. Persists each profile as a JSON file named after its
-    # token, under tmp_path/profiles by default or the directory given as options[:path], and
-    # evicts the oldest files when total size exceeds max_size. Only the files named after a
-    # token are taken for profiles, so a path shared with other files (tmp_path itself, where the
-    # env overrides live) is safe.
+    # token, under tmp_path/profiles by default or the directory given as options[:path]. Only the
+    # files named after a token are taken for profiles, so a path shared with other files (tmp_path
+    # itself, where the env overrides live) is safe.
+    #
+    # An index next to the profiles (INDEX_FILE) holds one line per save or delete: the token, the
+    # size of the file, the type, the parent and the summary the lists show. Every process keeps it
+    # in memory and reads only the lines appended since its last read, its own and those of the
+    # other processes writing to the directory (Puma workers, Sidekiq), so a save no longer lists
+    # or stats the directory. A save writes its file and its line under a shared flock; the
+    # compaction, which evicts the first saved profiles past max_profiles or max_size,
+    # resynchronizes the index with the directory (profiles written without it, files removed by
+    # hand, temporary files left by a killed writer) and rewrites it, holds the exclusive one: a
+    # save past a cap waits for it, a compaction of dead lines only gives up when another process
+    # holds it. A missing or damaged index is rebuilt from the files. flock has to work across
+    # the processes sharing the directory: one host, not NFS between hosts (see the README).
     class FileStore < BaseStore
+      INDEX_FILE = ".profiles-index.jsonl"
+      LOCK_FILE = ".profiles-index.lock"
+      INDEX_VERSION = 1
+      PROFILE_FILE = /\A(\h{32})\.json\z/
+      # The temporary files PrivateFiles.write makes for this store: a profile or the index, then
+      # the pid and a random part. Any other file of a shared directory is left alone.
+      TEMPORARY_FILE = /\A\.(?:\h{32}\.json|#{Regexp.escape(INDEX_FILE)})\.\d+\.\h{8}\.tmp\z/
+      # A temporary file older than this is one a killed writer left behind.
+      STALE_TEMPORARY_AGE = 60
+      # Past a cap, the oldest profiles are evicted down to this share of it.
+      LOW_WATER = 0.8
+
+      Entry = Struct.new(:token, :at, :type, :parent, :bytes, :summary, :sequence)
+
       def initialize(options = {})
         super()
         @max_size = options[:max_size] || (100 * 1024 * 1024) # 100 MB
+        @max_profiles = options.key?(:max_profiles) ? options[:max_profiles] : Profiler.configuration.max_profiles
         if options[:path]
           @path = options[:path].to_s
           PrivateFiles.mkdir(@path)
         else
           @path = PrivateFiles.tmp_dir("profiles").to_s
         end
+        @index_path = File.join(@path, INDEX_FILE)
+        @lock_path = File.join(@path, LOCK_FILE)
+        # Left with a wider mode by an earlier run: back to 0600 (a link is refused).
+        PrivateFiles.restrict(@lock_path)
+        PrivateFiles.restrict(@index_path)
+        @mutex = Mutex.new
+        forget_index
       end
 
       def do_save(token, profile)
-        PrivateFiles.write(profile_file_path(token), profile.to_json)
-        cleanup_if_needed
+        data = profile.to_h
+        json = data.to_json
+        line = JSON.generate(put_record(token, data, json.bytesize, started_at: profile.started_at))
+        @mutex.synchronize do
+          refresh if @generation.nil? # the index exists, with its header, before the first line
+          # The file and its line under one shared lock: a compaction sees both or neither, and
+          # cannot evict the file before its line is written.
+          with_lock(File::LOCK_SH) do
+            PrivateFiles.write(profile_file_path(token), json)
+            begin
+              PrivateFiles.append(@index_path, "#{line}\n")
+            rescue StandardError
+              # A file the index never learns of would be listed by no one and counted nowhere.
+              FileUtils.rm_f(profile_file_path(token))
+              raise
+            end
+          end
+          refresh
+          if over_cap?
+            compact(keep: token)
+          elsif compaction_due?
+            # Dead lines only: another process compacting meanwhile does it for this one.
+            compact(keep: token, wait: false)
+          end
+        end
         token
       end
 
@@ -45,55 +104,56 @@ module Profiler
         nil
       end
 
-      def list(limit: 50, offset: 0)
-        profile_files
-          .sort_by { |f| File.mtime(f) }
-          .reverse
-          .drop(offset)
-          .take(limit)
-          .map { |f| load(File.basename(f, ".json")) }
-          .compact
+      # Newest first. summary: true reads no profile file, only the index.
+      def list(limit: 50, offset: 0, type: nil, summary: false)
+        page = @mutex.synchronize do
+          refresh
+          entries = sorted_entries
+          entries = entries.select { |entry| entry.type == type.to_s } if type
+          entries.drop(offset).first(limit)
+        end
+        if summary
+          page.filter_map { |entry| Summary.to_profile(entry.summary) if entry.summary }
+        else
+          page.filter_map { |entry| load(entry.token) }
+        end
       end
 
+      # By the modification time of the files, as before the index: an explicit task, which may
+      # stat every profile.
       def cleanup(older_than: 24 * 60 * 60)
-        cutoff_time = Time.now - older_than
-        profile_files.each do |file|
-          File.delete(file) if File.mtime(file) < cutoff_time
-        rescue Errno::ENOENT
-          nil
+        cutoff = Time.now - older_than
+        @mutex.synchronize do
+          compact { |entries| entries.values.select { |entry| modified_before?(entry.token, cutoff) } }
         end
       end
 
       def find_by_parent(parent_token)
         return [] unless Token.valid?(parent_token)
 
-        profile_files
-          .map { |f| load(File.basename(f, ".json")) }
-          .compact
-          .select { |profile| profile.parent_token == parent_token }
-          .sort_by(&:started_at)
+        children = @mutex.synchronize do
+          refresh
+          (@children[parent_token] || {}).keys.filter_map { |token| @entries[token] }
+        end
+        children.filter_map { |entry| load(entry.token) }.sort_by(&:started_at)
       end
 
       def delete(token)
         return unless Token.valid?(token)
 
-        FileUtils.rm_f(profile_file_path(token))
+        @mutex.synchronize do
+          refresh if @generation.nil?
+          with_lock(File::LOCK_SH) do
+            FileUtils.rm_f(profile_file_path(token))
+            PrivateFiles.append(@index_path, "#{JSON.generate("op" => "del", "token" => token)}\n")
+          end
+          refresh
+        end
       end
 
-      def clear(type: nil) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        if type.nil?
-          profile_files.each do |f|
-            File.delete(f)
-          rescue StandardError
-            nil
-          end
-        else
-          profile_files.each do |f|
-            profile = load(File.basename(f, ".json"))
-            File.delete(f) if profile&.profile_type == type.to_s
-          rescue StandardError
-            nil
-          end
+      def clear(type: nil)
+        @mutex.synchronize do
+          compact { |entries| entries.values.select { |entry| type.nil? || entry.type == type.to_s } }
         end
       end
 
@@ -103,31 +163,245 @@ module Profiler
         File.join(@path, "#{token}.json")
       end
 
-      def profile_files
-        Dir.glob(File.join(@path, "*.json")).select { |f| Token.valid?(File.basename(f, ".json")) }
+      def modified_before?(token, cutoff)
+        File.mtime(profile_file_path(token)) < cutoff
+      rescue Errno::ENOENT
+        true
+      end
+
+      # started_at: the profile's own, more precise than the second of its JSON form.
+      def put_record(token, data, bytes, started_at: nil)
+        started_at = started_at&.to_f || (data[:started_at] ? Time.parse(data[:started_at]).to_f : Time.now.to_f)
+        { "op" => "put", "token" => token, "at" => started_at, "type" => (data[:profile_type] || "http").to_s,
+          "parent" => data[:parent_token], "bytes" => bytes, "summary" => Summary.build(data) }
+      end
+
+      def forget_index
+        @entries = {}
+        @sequence = 0
+        @children = {}
+        @bytes = 0
+        @records = 0
+        @offset = 0
+        @generation = nil
+        @damaged = false
+        @sorted = nil
       end
 
-      def cleanup_if_needed # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        total_size = profile_files.sum do |f|
-          File.size(f)
-        rescue Errno::ENOENT
-          0
+      # Brings the in-memory index up to date with the file: the appended lines only, or the whole
+      # file when another process rewrote it. A missing or damaged index is rebuilt.
+      def refresh
+        PrivateFiles.open_for_reading(@index_path) do |file|
+          header = parse_line(file.gets)
+          if header.nil? || header["profiler_index"] != INDEX_VERSION || !header["generation"].is_a?(String)
+            @damaged = true
+          elsif header["generation"] != @generation || file.size < @offset
+            forget_index
+            @generation = header["generation"]
+            @offset = file.pos
+            read_records(file)
+          elsif file.size > @offset
+            file.seek(@offset)
+            read_records(file)
+          end
         end
-        return if total_size < @max_size
+        compact if @damaged
+      rescue Errno::ENOENT
+        compact
+      end
+
+      # Consumes the complete lines from the file's position; a line still being written by
+      # another process is left for the next read.
+      def read_records(file)
+        data = file.read
+        complete = data.rindex("\n")
+        return unless complete
 
-        sorted = profile_files.sort_by do |f|
-          File.mtime(f)
-        rescue Errno::ENOENT
-          Time.now
+        data[0..complete].each_line do |line|
+          record = parse_line(line)
+          record ? apply(record) : @damaged = true
         end
+        @offset += data[0..complete].bytesize
+      end
+
+      def parse_line(line)
+        return nil if line.nil?
+
+        record = JSON.parse(line)
+        record.is_a?(Hash) ? record : nil
+      rescue JSON::ParserError
+        nil
+      end
+
+      # A record whose token is not one the gem issues is skipped: it is never turned into a path.
+      def apply(record)
+        token = record["token"]
+        return unless Token.valid?(token)
+
+        @records += 1
+        case record["op"]
+        when "put"
+          remove_entry(token)
+          parent = Token.valid?(record["parent"]) ? record["parent"] : nil
+          add_entry(Entry.new(token, record["at"].to_f, record["type"], parent, record["bytes"].to_i,
+                              record["summary"].is_a?(Hash) ? record["summary"] : nil))
+        when "del"
+          remove_entry(token)
+        end
+      end
+
+      def add_entry(entry)
+        entry.sequence = (@sequence += 1)
+        @entries[entry.token] = entry
+        @bytes += entry.bytes
+        (@children[entry.parent] ||= {})[entry.token] = true if entry.parent
+        @sorted = nil
+      end
+
+      def remove_entry(token)
+        entry = @entries.delete(token)
+        return unless entry
+
+        @bytes -= entry.bytes
+        if entry.parent && (siblings = @children[entry.parent])
+          siblings.delete(token)
+          @children.delete(entry.parent) if siblings.empty?
+        end
+        @sorted = nil
+      end
+
+      def sorted_entries
+        # Two profiles started in the same instant: the one saved last first.
+        @sorted ||= @entries.values.sort_by { |entry| [-entry.at, -entry.sequence] }
+      end
+
+      def over_cap?
+        (@max_profiles && @entries.size > @max_profiles) || @bytes >= @max_size
+      end
+
+      # The file holds more dead lines (replaced or deleted profiles) than live ones.
+      def compaction_due?
+        over_cap? || @records > (2 * @entries.size) + 64
+      end
+
+      # Under the exclusive lock: reads the whole index again, resynchronizes it with the
+      # directory, takes out the profiles the block returns (if any) and the first saved past the
+      # caps, never keep (the profile just saved), rewrites the index, then removes their files:
+      # a reader never lists a profile whose file is gone. Another process that compacted
+      # meanwhile leaves nothing to do but the rewrite. wait: false gives up when another process
+      # holds the lock.
+      def compact(keep: nil, wait: true)
+        with_lock(wait ? File::LOCK_EX : File::LOCK_EX | File::LOCK_NB) do
+          reload_for_compaction
+          resynchronize
+          doomed = (block_given? ? yield(@entries) : []).map(&:token)
+          doomed.each { |token| remove_entry(token) }
+          doomed.concat(evict_first_saved(keep)) if over_cap?
+          rewrite_index
+          doomed.each { |token| FileUtils.rm_f(profile_file_path(token)) }
+        end
+      end
+
+      # @last_compacted_at is kept only from a sound index: with a damaged one, a profile whose
+      # line was lost would pass for a file the last compaction evicted (see #resynchronize).
+      def reload_for_compaction
+        forget_index
+        @last_compacted_at = nil
+        PrivateFiles.open_for_reading(@index_path) do |file|
+          header = parse_line(file.gets)
+          next unless header && header["profiler_index"] == INDEX_VERSION && header["generation"].is_a?(String)
+
+          sound = true
+          file.each_line do |line|
+            record = parse_line(line)
+            record && line.end_with?("\n") ? apply(record) : sound = false
+          end
+          @last_compacted_at = header["compacted_at"] if sound && header["compacted_at"].is_a?(Numeric)
+        end
+      rescue Errno::ENOENT
+        nil
+      end
+
+      # The directory is the truth: entries whose file is gone are dropped, files the index does
+      # not know (written by an older version, or with the index lost) are read once and added,
+      # temporary files left by a killed writer are removed. A file the index does not know that
+      # is older than the last compaction was evicted by it (a save writes its file and its line
+      # under the lock, so a file saved before that compaction is in the index): the process that
+      # compacted was killed before it removed the file, which is removed now, not listed again.
+      def resynchronize
+        names = Dir.children(@path)
+        on_disk = names.filter_map { |name| name[PROFILE_FILE, 1] }
+        (@entries.keys - on_disk).each { |token| remove_entry(token) }
+        unknown = (on_disk - @entries.keys).map { |token| [token, modified_at(token)] }
+        evicted, unknown = unknown.partition { |_, mtime| @last_compacted_at && mtime < @last_compacted_at }
+        evicted.each { |token, _| FileUtils.rm_f(profile_file_path(token)) }
+        # Unknown files in the order they were saved, as far as their modification time tells.
+        unknown.sort_by(&:last).each { |token, _| add_entry(entry_from_file(token)) }
+        names.grep(TEMPORARY_FILE).each { |name| remove_stale_temporary(File.join(@path, name)) }
+      end
+
+      # A symbolic link in place of a profile is not followed while the modes are restricted.
+      def entry_from_file(token)
+        json = PrivateFiles.open_for_reading(profile_file_path(token), &:read)
+        record = put_record(token, Models::Profile.from_json(json).to_h, json.bytesize)
+        Entry.new(token, record["at"], record["type"], record["parent"], record["bytes"], record["summary"])
+      rescue StandardError
+        # Unreadable: kept in the count, so that it is evicted in turn; listed by no one.
+        stat = File.lstat(profile_file_path(token)) rescue nil
+        Entry.new(token, stat ? stat.mtime.to_f : 0.0, nil, nil, stat ? stat.size : 0, nil)
+      end
+
+      def modified_at(token)
+        File.lstat(profile_file_path(token)).mtime.to_f
+      rescue SystemCallError
+        0.0
+      end
+
+      def remove_stale_temporary(path)
+        File.delete(path) if File.lstat(path).mtime < Time.now - STALE_TEMPORARY_AGE
+      rescue SystemCallError
+        nil
+      end
+
+      # In the order they were saved, not started: a job saved when it ends is a new profile. The
+      # entries keep the order of the index lines, the order of the saves.
+      def evict_first_saved(keep)
+        count_target = @max_profiles && [(@max_profiles * LOW_WATER).floor, 1].max
+        bytes_target = @bytes >= @max_size ? @max_size * LOW_WATER : nil
+        evicted = []
+        @entries.values.each do |entry|
+          break unless (count_target && @entries.size > count_target) || (bytes_target && @bytes >= bytes_target)
+          next if entry.token == keep
+
+          remove_entry(entry.token)
+          evicted << entry.token
+        end
+        evicted
+      end
+
+      def rewrite_index
+        generation = SecureRandom.hex(8)
+        lines = [JSON.generate("profiler_index" => INDEX_VERSION, "generation" => generation,
+                               "compacted_at" => Time.now.to_f)]
+        @entries.each_value do |entry|
+          lines << JSON.generate("op" => "put", "token" => entry.token, "at" => entry.at, "type" => entry.type,
+                                 "parent" => entry.parent, "bytes" => entry.bytes, "summary" => entry.summary)
+        end
+        data = "#{lines.join("\n")}\n"
+        PrivateFiles.write(@index_path, data)
+        @generation = generation
+        @offset = data.bytesize
+        @records = @entries.size
+        @damaged = false
+      end
+
+      # flock on a file of its own, opened for each use: a lock taken through a descriptor shared
+      # with a forked process would not exclude it. The mutex excludes the threads of this one.
+      def with_lock(mode)
+        PrivateFiles.open(@lock_path) do |file|
+          next unless file.flock(mode)
 
-        sorted.each do |file|
-          file_size = File.size(file)
-          File.delete(file)
-          total_size -= file_size
-          break if total_size < @max_size * 0.8
-        rescue Errno::ENOENT
-          nil
+          yield
         end
       end
     end