rack-mini-profiler 2.0.1 → 2.2.0

data/lib/mini_profiler/profiling_methods.rb

@@ -7,7 +7,14 @@ module Rack
       def record_sql(query, elapsed_ms, params = nil)
         return unless current && current.current_timer
         c = current
-        c.current_timer.add_sql(query, elapsed_ms, c.page_struct, params, c.skip_backtrace, c.full_backtrace)
+        c.current_timer.add_sql(
+          redact_sql_queries? ? nil : query,
+          elapsed_ms,
+          c.page_struct,
+          redact_sql_queries? ? nil : params,
+          c.skip_backtrace,
+          c.full_backtrace
+        )
       end
 
       def start_step(name)

data/lib/mini_profiler/snapshots_transporter.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+class ::Rack::MiniProfiler::SnapshotsTransporter
+  @@transported_snapshots_count = 0
+  @@successful_http_requests_count = 0
+  @@failed_http_requests_count = 0
+
+  class << self
+    def transported_snapshots_count
+      @@transported_snapshots_count
+    end
+    def successful_http_requests_count
+      @@successful_http_requests_count
+    end
+    def failed_http_requests_count
+      @@failed_http_requests_count
+    end
+
+    def transport(snapshot)
+      @transporter ||= self.new(Rack::MiniProfiler.config)
+      @transporter.ship(snapshot)
+    end
+  end
+
+  attr_reader :buffer
+  attr_accessor :max_buffer_size
+
+  def initialize(config)
+    @uri = URI(config.snapshots_transport_destination_url)
+    @auth_key = config.snapshots_transport_auth_key
+    @thread = nil
+    @thread_mutex = Mutex.new
+    @buffer = []
+    @buffer_mutex = Mutex.new
+    @max_buffer_size = 100
+    @testing = false
+  end
+
+  def ship(snapshot)
+    @buffer_mutex.synchronize do
+      @buffer << snapshot
+      @buffer.shift if @buffer.size > @max_buffer_size
+    end
+    @thread_mutex.synchronize { start_thread }
+  end
+
+  def flush_buffer
+    buffer_content = @buffer_mutex.synchronize do
+      @buffer.dup if @buffer.size > 0
+    end
+    if buffer_content
+      request = Net::HTTP::Post.new(
+        @uri,
+        'Content-Type' => 'application/json',
+        'Mini-Profiler-Transport-Auth' => @auth_key
+      )
+      request.body = { snapshots: buffer_content }.to_json
+      http = Net::HTTP.new(@uri.hostname, @uri.port)
+      http.use_ssl = @uri.scheme == 'https'
+      res = http.request(request)
+      if res.code.to_i == 200
+        @@successful_http_requests_count += 1
+        @@transported_snapshots_count += buffer_content.size
+        @buffer_mutex.synchronize do
+          @buffer -= buffer_content
+        end
+      else
+        @@failed_http_requests_count += 1
+      end
+    end
+  end
+
+  private
+
+  def start_thread
+    return if @thread&.alive? || @testing
+    @thread = Thread.new do
+      while true
+        sleep 10
+        flush_buffer
+      end
+    end
+  end
+end

data/lib/mini_profiler/storage/abstract_store.rb

@@ -41,6 +41,84 @@ module Rack
         raise NotImplementedError.new("allowed_tokens is not implemented")
       end
 
+      def should_take_snapshot?(period)
+        raise NotImplementedError.new("should_take_snapshot? is not implemented")
+      end
+
+      def push_snapshot(page_struct, config)
+        raise NotImplementedError.new("push_snapshot is not implemented")
+      end
+
+      def fetch_snapshots(batch_size: 200, &blk)
+        raise NotImplementedError.new("fetch_snapshots is not implemented")
+      end
+
+      def snapshot_groups_overview
+        groups = {}
+        fetch_snapshots do |batch|
+          batch.each do |snapshot|
+            group_name = default_snapshot_grouping(snapshot)
+            hash = groups[group_name] ||= {}
+            hash[:snapshots_count] ||= 0
+            hash[:snapshots_count] += 1
+            if !hash[:worst_score] || hash[:worst_score] < snapshot.duration_ms
+              groups[group_name][:worst_score] = snapshot.duration_ms
+            end
+            if !hash[:best_score] || hash[:best_score] > snapshot.duration_ms
+              groups[group_name][:best_score] = snapshot.duration_ms
+            end
+          end
+        end
+        groups = groups.to_a
+        groups.sort_by! { |name, hash| hash[:worst_score] }
+        groups.reverse!
+        groups.map! { |name, hash| hash.merge(name: name) }
+        groups
+      end
+
+      def find_snapshots_group(group_name)
+        data = []
+        fetch_snapshots do |batch|
+          batch.each do |snapshot|
+            snapshot_group_name = default_snapshot_grouping(snapshot)
+            if group_name == snapshot_group_name
+              data << {
+                id: snapshot[:id],
+                duration: snapshot.duration_ms,
+                sql_count: snapshot[:sql_count],
+                timestamp: snapshot[:started_at],
+                custom_fields: snapshot[:custom_fields]
+              }
+            end
+          end
+        end
+        data.sort_by! { |s| s[:duration] }
+        data.reverse!
+        data
+      end
+
+      def load_snapshot(id)
+        raise NotImplementedError.new("load_snapshot is not implemented")
+      end
+
+      private
+
+      def default_snapshot_grouping(snapshot)
+        group_name = rails_route_from_path(snapshot[:request_path], snapshot[:request_method])
+        group_name ||= snapshot[:request_path]
+        "#{snapshot[:request_method]} #{group_name}"
+      end
+
+      def rails_route_from_path(path, method)
+        if defined?(Rails) && defined?(ActionController::RoutingError)
+          hash = Rails.application.routes.recognize_path(path, method: method)
+          if hash && hash[:controller] && hash[:action]
+            "#{hash[:controller]}##{hash[:action]}"
+          end
+        end
+      rescue ActionController::RoutingError
+        nil
+      end
     end
   end
 end

data/lib/mini_profiler/storage/memory_store.rb

@@ -52,17 +52,21 @@ module Rack
         @expires_in_seconds = args.fetch(:expires_in) { EXPIRES_IN_SECONDS }
 
         @token1, @token2, @cycle_at = nil
+        @snapshots_cycle = 0
+        @snapshots = []
 
         initialize_locks
         initialize_cleanup_thread(args)
       end
 
       def initialize_locks
-        @token_lock         = Mutex.new
-        @timer_struct_lock  = Mutex.new
-        @user_view_lock     = Mutex.new
-        @timer_struct_cache = {}
-        @user_view_cache    = {}
+        @token_lock           = Mutex.new
+        @timer_struct_lock    = Mutex.new
+        @user_view_lock       = Mutex.new
+        @snapshots_cycle_lock = Mutex.new
+        @snapshots_lock       = Mutex.new
+        @timer_struct_cache   = {}
+        @user_view_cache      = {}
       end
 
       #FIXME: use weak ref, trouble it may be broken in 1.9 so need to use the 'ref' gem
@@ -135,6 +139,51 @@ module Rack
 
         end
       end
+
+      def should_take_snapshot?(period)
+        @snapshots_cycle_lock.synchronize do
+          @snapshots_cycle += 1
+          if @snapshots_cycle % period == 0
+            @snapshots_cycle = 0
+            true
+          else
+            false
+          end
+        end
+      end
+
+      def push_snapshot(page_struct, config)
+        @snapshots_lock.synchronize do
+          @snapshots << page_struct
+          @snapshots.sort_by! { |s| s.duration_ms }
+          @snapshots.reverse!
+          if @snapshots.size > config.snapshots_limit
+            @snapshots.slice!(-1)
+          end
+        end
+      end
+
+      def fetch_snapshots(batch_size: 200, &blk)
+        @snapshots_lock.synchronize do
+          @snapshots.each_slice(batch_size) do |batch|
+            blk.call(batch)
+          end
+        end
+      end
+
+      def load_snapshot(id)
+        @snapshots_lock.synchronize do
+          @snapshots.find { |s| s[:id] == id }
+        end
+      end
+
+      private
+
+      # used in tests only
+      def wipe_snapshots_data
+        @snapshots_cycle = 0
+        @snapshots = []
+      end
     end
   end
 end

data/lib/mini_profiler/storage/redis_store.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'digest'
+
 module Rack
   class MiniProfiler
     class RedisStore < AbstractStore
@@ -33,7 +35,7 @@ module Rack
 
       def set_unviewed(user, id)
         key = user_key(user)
-        if redis.exists(prefixed_id(id))
+        if redis.call([:exists, prefixed_id(id)]) == 1
           expire_at = Process.clock_gettime(Process::CLOCK_MONOTONIC).to_i + redis.ttl(prefixed_id(id))
           redis.zadd(key, expire_at, id)
         end
@@ -44,7 +46,7 @@ module Rack
         key = user_key(user)
         redis.del(key)
         ids.each do |id|
-          if redis.exists(prefixed_id(id))
+          if redis.call([:exists, prefixed_id(id)]) == 1
             expire_at = Process.clock_gettime(Process::CLOCK_MONOTONIC).to_i + redis.ttl(prefixed_id(id))
             redis.zadd(key, expire_at, id)
           end
@@ -108,6 +110,106 @@ unviewed_ids: #{get_unviewed_ids(user)}
         [key1, key2].compact
       end
 
+      COUNTER_LUA = <<~LUA
+        if redis.call("INCR", KEYS[1]) % ARGV[1] == 0 then
+          redis.call("DEL", KEYS[1])
+          return 1
+        else
+          return 0
+        end
+      LUA
+
+      COUNTER_LUA_SHA = Digest::SHA1.hexdigest(COUNTER_LUA)
+
+      def should_take_snapshot?(period)
+        1 == cached_redis_eval(
+          COUNTER_LUA,
+          COUNTER_LUA_SHA,
+          reraise: false,
+          keys: [snapshot_counter_key()],
+          argv: [period]
+        )
+      end
+
+      def push_snapshot(page_struct, config)
+        zset_key = snapshot_zset_key()
+        hash_key = snapshot_hash_key()
+
+        id = page_struct[:id]
+        score = page_struct.duration_ms
+        limit = config.snapshots_limit
+        bytes = Marshal.dump(page_struct)
+
+        lua = <<~LUA
+          local zset_key = KEYS[1]
+          local hash_key = KEYS[2]
+          local id = ARGV[1]
+          local score = tonumber(ARGV[2])
+          local bytes = ARGV[3]
+          local limit = tonumber(ARGV[4])
+          redis.call("ZADD", zset_key, score, id)
+          redis.call("HSET", hash_key, id, bytes)
+          if redis.call("ZCARD", zset_key) > limit then
+            local lowest_snapshot_id = redis.call("ZRANGE", zset_key, 0, 0)[1]
+            redis.call("ZREM", zset_key, lowest_snapshot_id)
+            redis.call("HDEL", hash_key, lowest_snapshot_id)
+          end
+        LUA
+        redis.eval(
+          lua,
+          keys: [zset_key, hash_key],
+          argv: [id, score, bytes, limit]
+        )
+      end
+
+      def fetch_snapshots(batch_size: 200, &blk)
+        zset_key = snapshot_zset_key()
+        hash_key = snapshot_hash_key()
+        iteration = 0
+        corrupt_snapshots = []
+        while true
+          ids = redis.zrange(
+            zset_key,
+            batch_size * iteration,
+            batch_size * iteration + batch_size - 1
+          )
+          break if ids.size == 0
+          batch = redis.mapped_hmget(hash_key, *ids).to_a
+          batch.map! do |id, bytes|
+            begin
+              Marshal.load(bytes)
+            rescue
+              corrupt_snapshots << id
+              nil
+            end
+          end
+          batch.compact!
+          blk.call(batch) if batch.size != 0
+          break if ids.size < batch_size
+          iteration += 1
+        end
+        if corrupt_snapshots.size > 0
+          redis.pipelined do
+            redis.zrem(zset_key, corrupt_snapshots)
+            redis.hdel(hash_key, corrupt_snapshots)
+          end
+        end
+      end
+
+      def load_snapshot(id)
+        hash_key = snapshot_hash_key()
+        bytes = redis.hget(hash_key, id)
+        begin
+          Marshal.load(bytes)
+        rescue
+          redis.pipelined do
+            redis.zrem(snapshot_zset_key(), id)
+            redis.hdel(hash_key, id)
+          end
+          nil
+        end
+      end
+
       private
 
       def user_key(user)
@@ -125,6 +227,38 @@ unviewed_ids: #{get_unviewed_ids(user)}
         end
       end
 
+      def snapshot_counter_key
+        @snapshot_counter_key ||= "#{@prefix}-mini-profiler-snapshots-counter"
+      end
+
+      def snapshot_zset_key
+        @snapshot_zset_key ||= "#{@prefix}-mini-profiler-snapshots-zset"
+      end
+
+      def snapshot_hash_key
+        @snapshot_hash_key ||= "#{@prefix}-mini-profiler-snapshots-hash"
+      end
+
+      def cached_redis_eval(script, script_sha, reraise: true, argv: [], keys: [])
+        begin
+          redis.evalsha(script_sha, argv: argv, keys: keys)
+        rescue ::Redis::CommandError => e
+          if e.message.start_with?('NOSCRIPT')
+            redis.eval(script, argv: argv, keys: keys)
+          else
+            raise e if reraise
+          end
+        end
+      end
+
+      # only used in tests
+      def wipe_snapshots_data
+        redis.pipelined do
+          redis.del(snapshot_counter_key())
+          redis.del(snapshot_zset_key())
+          redis.del(snapshot_hash_key())
+        end
+      end
     end
   end
 end

data/lib/mini_profiler/timer_struct/page.rb

@@ -10,6 +10,53 @@ module Rack
       #     :has_many TimerStruct::Sql children
       #     :has_many TimerStruct::Custom children
       class Page < TimerStruct::Base
+        class << self
+          def from_hash(hash)
+            hash = symbolize_hash(hash)
+            if hash.key?(:custom_timing_names)
+              hash[:custom_timing_names] = []
+            end
+            hash.delete(:started_formatted)
+            if hash.key?(:duration_milliseconds)
+              hash[:duration_milliseconds] = 0
+            end
+            page = self.allocate
+            page.instance_variable_set(:@attributes, hash)
+            page
+          end
+
+          private
+
+          def symbolize_hash(hash)
+            new_hash = {}
+            hash.each do |k, v|
+              sym_k = String === k ? k.to_sym : k
+              if Hash === v
+                new_hash[sym_k] = symbolize_hash(v)
+              elsif Array === v
+                new_hash[sym_k] = symbolize_array(v)
+              else
+                new_hash[sym_k] = v
+              end
+            end
+            new_hash
+          end
+
+          def symbolize_array(array)
+            array.map do |item|
+              if Array === item
+                symbolize_array(item)
+              elsif Hash === item
+                symbolize_hash(item)
+              else
+                item
+              end
+            end
+          end
+        end
+
+        attr_reader :attributes
+
         def initialize(env)
           timer_id     = MiniProfiler.generate_id
           page_name    = env['PATH_INFO']
@@ -39,8 +86,11 @@ module Rack
             executed_scalars: 0,
             executed_non_queries: 0,
             custom_timing_names: [],
-            custom_timing_stats: {}
+            custom_timing_stats: {},
+            custom_fields: {}
           )
+          self[:request_method] = env['REQUEST_METHOD']
+          self[:request_path] = env['PATH_INFO']
           name = "#{env['REQUEST_METHOD']} http://#{env['SERVER_NAME']}:#{env['SERVER_PORT']}#{env['SCRIPT_NAME']}#{env['PATH_INFO']}"
           self[:root] = TimerStruct::Request.createRoot(name, self)
         end
@@ -71,7 +121,7 @@ module Rack
 
         def extra_json
           {
-            started: '/Date(%d)/' % @attributes[:started_at],
+            started_formatted: '/Date(%d)/' % @attributes[:started_at],
             duration_milliseconds: @attributes[:root][:duration_milliseconds],
             custom_timing_names: @attributes[:custom_timing_stats].keys.sort
           }