rack-mini-profiler 2.1.0 → 2.3.2

data/lib/mini_profiler/config.rb

@@ -52,6 +52,11 @@ module Rack
           @toggle_shortcut      = 'alt+p'
           @html_container       = 'body'
           @position             = "top-left"
+          @snapshot_hidden_custom_fields = []
+          @snapshots_transport_destination_url = nil
+          @snapshots_transport_auth_key = nil
+          @snapshots_redact_sql_queries = true
+          @snapshots_transport_gzip_requests = false
 
           self
         }
@@ -63,19 +68,40 @@ module Rack
         :flamegraph_sample_rate, :logger, :pre_authorize_cb, :skip_paths,
         :skip_schema_queries, :storage, :storage_failure, :storage_instance,
         :storage_options, :user_provider, :enable_advanced_debugging_tools,
-        :snapshot_every_n_requests, :snapshots_limit
-      attr_accessor :skip_sql_param_names, :suppress_encoding, :max_sql_param_length
+        :skip_sql_param_names, :suppress_encoding, :max_sql_param_length
 
       # ui accessors
       attr_accessor :collapse_results, :max_traces_to_show, :position,
         :show_children, :show_controls, :show_trivial, :show_total_sql_count,
         :start_hidden, :toggle_shortcut, :html_container
 
+      # snapshot related config
+      attr_accessor :snapshot_every_n_requests, :snapshots_limit,
+        :snapshot_hidden_custom_fields, :snapshots_transport_destination_url,
+        :snapshots_transport_auth_key, :snapshots_redact_sql_queries,
+        :snapshots_transport_gzip_requests
+
       # Deprecated options
       attr_accessor :use_existing_jquery
 
       attr_reader :assets_url
 
+      # redefined - since the accessor defines it first
+      undef :authorization_mode=
+      def authorization_mode=(mode)
+        if mode == :whitelist
+          warn "[DEPRECATION] `:whitelist` authorization mode is deprecated. Please use `:allow_authorized` instead."
+
+          mode = :allow_authorized
+        end
+
+        warn <<~DEP unless mode == :allow_authorized || mode == :allow_all
+          [DEPRECATION] unknown authorization mode #{mode}. Expected `:allow_all` or `:allow_authorized`.
+        DEP
+
+        @authorization_mode = mode
+      end
+
       def assets_url=(lmbda)
         if defined?(Rack::MiniProfilerRails)
           Rack::MiniProfilerRails.create_engine

data/lib/mini_profiler/profiler.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'cgi'
+
 module Rack
   class MiniProfiler
     class << self
@@ -39,6 +41,7 @@ module Rack
       def current=(c)
         # we use TLS cause we need access to this from sql blocks and code blocks that have no access to env
         Thread.current[:mini_profiler_snapshot_custom_fields] = nil
+        Thread.current[:mp_ongoing_snapshot] = nil
         Thread.current[:mini_profiler_private] = c
       end
 
@@ -94,6 +97,16 @@ module Rack
           params
         end
       end
+
+      def snapshots_transporter?
+        !!config.snapshots_transport_destination_url &&
+        !!config.snapshots_transport_auth_key
+      end
+
+      def redact_sql_queries?
+        Thread.current[:mp_ongoing_snapshot] == true &&
+        Rack::MiniProfiler.config.snapshots_redact_sql_queries
+      end
     end
 
     #
@@ -169,6 +182,7 @@ module Rack
 
       return serve_results(env) if file_name.eql?('results')
       return handle_snapshots_request(env) if file_name.eql?('snapshots')
+      return serve_flamegraph(env) if file_name.eql?('flamegraph')
 
       resources_env = env.dup
       resources_env['PATH_INFO'] = file_name
@@ -200,7 +214,7 @@ module Rack
     def call(env)
       start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       client_settings = ClientSettings.new(env, @storage, start)
-      MiniProfiler.deauthorize_request if @config.authorization_mode == :whitelist
+      MiniProfiler.deauthorize_request if @config.authorization_mode == :allow_authorized
 
       status = headers = body = nil
       query_string = env['QUERY_STRING']
@@ -226,7 +240,7 @@ module Rack
       skip_it = (@config.pre_authorize_cb && !@config.pre_authorize_cb.call(env))
 
       if skip_it || (
-        @config.authorization_mode == :whitelist &&
+        @config.authorization_mode == :allow_authorized &&
         !client_settings.has_valid_cookie?
       )
         if take_snapshot?(path)
@@ -268,6 +282,15 @@ module Rack
       # profile memory
       if query_string =~ /pp=profile-memory/
         return tool_disabled_message(client_settings) if !advanced_debugging_enabled?
+
+        unless defined?(MemoryProfiler) && MemoryProfiler.respond_to?(:report)
+          message = "Please install the memory_profiler gem and require it: add gem 'memory_profiler' to your Gemfile"
+          _, _, body = @app.call(env)
+          body.close if body.respond_to? :close
+
+          return client_settings.handle_cookie(text_result(message))
+        end
+
         query_params = Rack::Utils.parse_nested_query(query_string)
         options = {
           ignore_files: query_params['memory_profiler_ignore_files'],
@@ -323,24 +346,28 @@ module Rack
         # Prevent response body from being compressed
         env['HTTP_ACCEPT_ENCODING'] = 'identity' if config.suppress_encoding
 
-        if query_string =~ /pp=flamegraph/
-          unless defined?(Flamegraph) && Flamegraph.respond_to?(:generate)
-
-            flamegraph = "Please install the flamegraph gem and require it: add gem 'flamegraph' to your Gemfile"
-            status, headers, body = @app.call(env)
+        if query_string =~ /pp=(async-)?flamegraph/ || env['HTTP_REFERER'] =~ /pp=async-flamegraph/
+          unless defined?(StackProf) && StackProf.respond_to?(:run)
+            headers = { 'Content-Type' => 'text/html' }
+            message = "Please install the stackprof gem and require it: add gem 'stackprof' to your Gemfile"
+            body.close if body.respond_to? :close
+            return client_settings.handle_cookie([500, headers, message])
           else
             # do not sully our profile with mini profiler timings
             current.measure = false
             match_data      = query_string.match(/flamegraph_sample_rate=([\d\.]+)/)
 
-            mode = query_string =~ /mode=c/ ? :c : :ruby
-
             if match_data && !match_data[1].to_f.zero?
               sample_rate = match_data[1].to_f
             else
               sample_rate = config.flamegraph_sample_rate
             end
-            flamegraph = Flamegraph.generate(nil, fidelity: sample_rate, embed_resources: query_string =~ /embed/, mode: mode) do
+            flamegraph = StackProf.run(
+              mode: :wall,
+              raw: true,
+              aggregate: false,
+              interval: (sample_rate * 1000).to_i
+            ) do
               status, headers, body = @app.call(env)
             end
           end
@@ -363,7 +390,7 @@ module Rack
 
       skip_it = current.discard
 
-      if (config.authorization_mode == :whitelist && !MiniProfiler.request_authorized?)
+      if (config.authorization_mode == :allow_authorized && !MiniProfiler.request_authorized?)
         skip_it = true
       end
 
@@ -404,9 +431,12 @@ module Rack
       page_struct[:user] = user(env)
       page_struct[:root].record_time((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000)
 
-      if flamegraph
+      if flamegraph && query_string =~ /pp=flamegraph/
         body.close if body.respond_to? :close
-        return client_settings.handle_cookie(self.flamegraph(flamegraph))
+        return client_settings.handle_cookie(self.flamegraph(flamegraph, path))
+      elsif flamegraph # async-flamegraph
+        page_struct[:has_flamegraph] = true
+        page_struct[:flamegraph] = flamegraph
       end
 
       begin
@@ -623,13 +653,14 @@ Append the following to your query string:
   #{make_link "full-backtrace", env} #{"(*) " if client_settings.backtrace_full?}: enable full backtraces for SQL executed (use pp=normal-backtrace to disable)
   #{make_link "disable", env} : disable profiling for this session
   #{make_link "enable", env} : enable profiling for this session (if previously disabled)
-  #{make_link "profile-gc", env} : perform gc profiling on this request, analyzes ObjectSpace generated by request (ruby 1.9.3 only)
+  #{make_link "profile-gc", env} : perform gc profiling on this request, analyzes ObjectSpace generated by request
   #{make_link "profile-memory", env} : requires the memory_profiler gem, new location based report
-  #{make_link "flamegraph", env} : works best on Ruby 2.0, a graph representing sampled activity (requires the flamegraph gem).
+  #{make_link "flamegraph", env} : a graph representing sampled activity (requires the stackprof gem).
+  #{make_link "async-flamegraph", env} : store flamegraph data for this page and all its AJAX requests. Flamegraph links will be available in the mini-profiler UI (requires the stackprof gem).
   #{make_link "flamegraph&flamegraph_sample_rate=1", env}: creates a flamegraph with the specified sample rate (in ms). Overrides value set in config
-  #{make_link "flamegraph_embed", env} : works best on Ruby 2.0, a graph representing sampled activity (requires the flamegraph gem), embedded resources for use on an intranet.
-  #{make_link "trace-exceptions", env} : requires Ruby 2.0, will return all the spots where your application raises exceptions
-  #{make_link "analyze-memory", env} : requires Ruby 2.0, will perform basic memory analysis of heap
+  #{make_link "flamegraph_embed", env} : a graph representing sampled activity (requires the stackprof gem), embedded resources for use on an intranet.
+  #{make_link "trace-exceptions", env} : will return all the spots where your application raises exceptions
+  #{make_link "analyze-memory", env} : will perform basic memory analysis of heap
 </pre>
 </body>
 </html>
@@ -638,9 +669,33 @@ Append the following to your query string:
       [200, headers, [body]]
     end
 
-    def flamegraph(graph)
+    def flamegraph(graph, path)
       headers = { 'Content-Type' => 'text/html' }
-      [200, headers, [graph]]
+      html = <<~HTML
+        <!DOCTYPE html>
+        <html>
+          <head>
+            <style>
+              body { margin: 0; height: 100vh; }
+              #speedscope-iframe { width: 100%; height: 100%; border: none; }
+            </style>
+          </head>
+          <body>
+            <script type="text/javascript">
+              var graph = #{JSON.generate(graph)};
+              var json = JSON.stringify(graph);
+              var blob = new Blob([json], { type: 'text/plain' });
+              var objUrl = encodeURIComponent(URL.createObjectURL(blob));
+              var iframe = document.createElement('IFRAME');
+              iframe.setAttribute('id', 'speedscope-iframe');
+              document.body.appendChild(iframe);
+              var iframeUrl = '#{@config.base_url_path}speedscope/index.html#profileURL=' + objUrl + '&title=' + 'Flamegraph for #{CGI.escape(path)}';
+              iframe.setAttribute('src', iframeUrl);
+            </script>
+          </body>
+        </html>
+      HTML
+      [200, headers, [html]]
     end
 
     def ids(env)
@@ -689,7 +744,8 @@ Append the following to your query string:
        toggleShortcut: @config.toggle_shortcut,
        startHidden: @config.start_hidden,
        collapseResults: @config.collapse_results,
-       htmlContainer: @config.html_container
+       htmlContainer: @config.html_container,
+       hiddenCustomFields: @config.snapshot_hidden_custom_fields.join(',')
       }
 
       if current && current.page_struct
@@ -770,6 +826,24 @@ Append the following to your query string:
       response.finish
     end
 
+    def serve_flamegraph(env)
+      request     = Rack::Request.new(env)
+      id          = request.params['id']
+      page_struct = @storage.load(id)
+
+      if !page_struct
+        id        = ERB::Util.html_escape(id)
+        user_info = ERB::Util.html_escape(user(env))
+        return [404, {}, ["Request not found: #{id} - user #{user_info}"]]
+      end
+
+      if !page_struct[:flamegraph]
+        return [404, {}, ["No flamegraph available for #{ERB::Util.html_escape(id)}"]]
+      end
+
+      self.flamegraph(page_struct[:flamegraph], page_struct[:request_path])
+    end
+
     def rails_route_from_path(path, method)
       if defined?(Rails) && defined?(ActionController::RoutingError)
         hash = Rails.application.routes.recognize_path(path, method: method)
@@ -799,6 +873,7 @@ Append the following to your query string:
 
     def take_snapshot(env, start)
       MiniProfiler.create_current(env, @config)
+      Thread.current[:mp_ongoing_snapshot] = true
       results = @app.call(env)
       status = results[0].to_i
       if status >= 200 && status < 300
@@ -808,10 +883,14 @@ Append the following to your query string:
         )
         custom_fields = MiniProfiler.get_snapshot_custom_fields
         page_struct[:custom_fields] = custom_fields if custom_fields
-        @storage.push_snapshot(
-          page_struct,
-          @config
-        )
+        if Rack::MiniProfiler.snapshots_transporter?
+          Rack::MiniProfiler::SnapshotsTransporter.transport(page_struct)
+        else
+          @storage.push_snapshot(
+            page_struct,
+            @config
+          )
+        end
       end
       self.current = nil
       results

data/lib/mini_profiler/profiling_methods.rb

@@ -7,7 +7,14 @@ module Rack
       def record_sql(query, elapsed_ms, params = nil)
         return unless current && current.current_timer
         c = current
-        c.current_timer.add_sql(query, elapsed_ms, c.page_struct, params, c.skip_backtrace, c.full_backtrace)
+        c.current_timer.add_sql(
+          redact_sql_queries? ? nil : query,
+          elapsed_ms,
+          c.page_struct,
+          redact_sql_queries? ? nil : params,
+          c.skip_backtrace,
+          c.full_backtrace
+        )
       end
 
       def start_step(name)
@@ -108,6 +115,9 @@ module Rack
             end
           end
         end
+        if klass.respond_to?(:ruby2_keywords, true)
+          klass.send(:ruby2_keywords, with_profiling)
+        end
         klass.send :alias_method, method, with_profiling
       end
 
@@ -147,7 +157,6 @@ module Rack
       def clean_method_name(method)
         method.to_s.gsub(/[\?\!]/, "")
       end
-
     end
   end
 end

data/lib/mini_profiler/snapshots_transporter.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+class ::Rack::MiniProfiler::SnapshotsTransporter
+  @@transported_snapshots_count = 0
+  @@successful_http_requests_count = 0
+  @@failed_http_requests_count = 0
+
+  class << self
+    def transported_snapshots_count
+      @@transported_snapshots_count
+    end
+    def successful_http_requests_count
+      @@successful_http_requests_count
+    end
+    def failed_http_requests_count
+      @@failed_http_requests_count
+    end
+
+    def transport(snapshot)
+      @transporter ||= self.new(Rack::MiniProfiler.config)
+      @transporter.ship(snapshot)
+    end
+  end
+
+  attr_reader :buffer
+  attr_accessor :max_buffer_size, :gzip_requests
+
+  def initialize(config)
+    @uri = URI(config.snapshots_transport_destination_url)
+    @auth_key = config.snapshots_transport_auth_key
+    @gzip_requests = config.snapshots_transport_gzip_requests
+    @thread = nil
+    @thread_mutex = Mutex.new
+    @buffer = []
+    @buffer_mutex = Mutex.new
+    @max_buffer_size = 100
+    @consecutive_failures_count = 0
+    @testing = false
+  end
+
+  def ship(snapshot)
+    @buffer_mutex.synchronize do
+      @buffer << snapshot
+      @buffer.shift if @buffer.size > @max_buffer_size
+    end
+    @thread_mutex.synchronize { start_thread }
+  end
+
+  def flush_buffer
+    buffer_content = @buffer_mutex.synchronize do
+      @buffer.dup if @buffer.size > 0
+    end
+    if buffer_content
+      headers = {
+        'Content-Type' => 'application/json',
+        'Mini-Profiler-Transport-Auth' => @auth_key
+      }
+      json = { snapshots: buffer_content }.to_json
+      body = if @gzip_requests
+        require 'zlib'
+        io = StringIO.new
+        gzip_writer = Zlib::GzipWriter.new(io)
+        gzip_writer.write(json)
+        gzip_writer.close
+        headers['Content-Encoding'] = 'gzip'
+        io.string
+      else
+        json
+      end
+      request = Net::HTTP::Post.new(@uri, headers)
+      request.body = body
+      http = Net::HTTP.new(@uri.hostname, @uri.port)
+      http.use_ssl = @uri.scheme == 'https'
+      res = http.request(request)
+      if res.code.to_i == 200
+        @@successful_http_requests_count += 1
+        @@transported_snapshots_count += buffer_content.size
+        @buffer_mutex.synchronize do
+          @buffer -= buffer_content
+        end
+        @consecutive_failures_count = 0
+      else
+        @@failed_http_requests_count += 1
+        @consecutive_failures_count += 1
+      end
+    end
+  end
+
+  def requests_interval
+    [30 + backoff_delay, 60 * 60].min
+  end
+
+  private
+
+  def backoff_delay
+    return 0 if @consecutive_failures_count == 0
+    2**@consecutive_failures_count
+  end
+
+  def start_thread
+    return if @thread&.alive? || @testing
+    @thread = Thread.new do
+      while true
+        sleep requests_interval
+        flush_buffer
+      end
+    end
+  end
+end

data/lib/mini_profiler/storage/abstract_store.rb

@@ -36,7 +36,7 @@ module Rack
         ""
       end
 
-      # a list of tokens that are permitted to access profiler in whitelist mode
+      # a list of tokens that are permitted to access profiler in explicit mode
       def allowed_tokens
         raise NotImplementedError.new("allowed_tokens is not implemented")
       end
@@ -58,17 +58,21 @@ module Rack
         fetch_snapshots do |batch|
           batch.each do |snapshot|
             group_name = default_snapshot_grouping(snapshot)
-            if !groups[group_name] || groups[group_name] < snapshot.duration_ms
-              groups[group_name] = snapshot.duration_ms
+            hash = groups[group_name] ||= {}
+            hash[:snapshots_count] ||= 0
+            hash[:snapshots_count] += 1
+            if !hash[:worst_score] || hash[:worst_score] < snapshot.duration_ms
+              groups[group_name][:worst_score] = snapshot.duration_ms
+            end
+            if !hash[:best_score] || hash[:best_score] > snapshot.duration_ms
+              groups[group_name][:best_score] = snapshot.duration_ms
             end
           end
         end
         groups = groups.to_a
-        groups.sort_by! { |name, score| score }
+        groups.sort_by! { |name, hash| hash[:worst_score] }
         groups.reverse!
-        groups.map! do |name, score|
-          { name: name, worst_score: score }
-        end
+        groups.map! { |name, hash| hash.merge(name: name) }
         groups
       end
 
@@ -81,7 +85,9 @@ module Rack
               data << {
                 id: snapshot[:id],
                 duration: snapshot.duration_ms,
-                timestamp: snapshot[:started_at]
+                sql_count: snapshot[:sql_count],
+                timestamp: snapshot[:started_at],
+                custom_fields: snapshot[:custom_fields]
               }
             end
           end