rack-mini-profiler 1.0.2 → 3.1.0

data/lib/mini_profiler/storage/redis_store.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'digest'
+require 'securerandom'
+
 module Rack
   class MiniProfiler
     class RedisStore < AbstractStore
@@ -23,7 +26,9 @@ module Rack
         key = prefixed_id(id)
         raw = redis.get key
         begin
-          Marshal::load(raw) if raw
+          # rubocop:disable Security/MarshalLoad
+          Marshal.load(raw) if raw
+          # rubocop:enable Security/MarshalLoad
         rescue
           # bad format, junk old data
           redis.del key
@@ -33,7 +38,7 @@ module Rack
 
       def set_unviewed(user, id)
         key = user_key(user)
-        if redis.exists(prefixed_id(id))
+        if redis.call([:exists, prefixed_id(id)]) == 1
           expire_at = Process.clock_gettime(Process::CLOCK_MONOTONIC).to_i + redis.ttl(prefixed_id(id))
           redis.zadd(key, expire_at, id)
         end
@@ -44,7 +49,7 @@ module Rack
         key = user_key(user)
         redis.del(key)
         ids.each do |id|
-          if redis.exists(prefixed_id(id))
+          if redis.call([:exists, prefixed_id(id)]) == 1
             expire_at = Process.clock_gettime(Process::CLOCK_MONOTONIC).to_i + redis.ttl(prefixed_id(id))
             redis.zadd(key, expire_at, id)
           end
@@ -108,6 +113,152 @@ unviewed_ids: #{get_unviewed_ids(user)}
         [key1, key2].compact
       end
 
+      COUNTER_LUA = <<~LUA
+        if redis.call("INCR", KEYS[1]) % ARGV[1] == 0 then
+          redis.call("DEL", KEYS[1])
+          return 1
+        else
+          return 0
+        end
+      LUA
+
+      COUNTER_LUA_SHA = Digest::SHA1.hexdigest(COUNTER_LUA)
+
+      def should_take_snapshot?(period)
+        1 == cached_redis_eval(
+          COUNTER_LUA,
+          COUNTER_LUA_SHA,
+          reraise: false,
+          keys: [snapshot_counter_key()],
+          argv: [period]
+        )
+      end
+
+      def push_snapshot(page_struct, group_name, config)
+        group_zset_key = group_snapshot_zset_key(group_name)
+        group_hash_key = group_snapshot_hash_key(group_name)
+        overview_zset_key = snapshot_overview_zset_key
+
+        id = page_struct[:id]
+        score = page_struct.duration_ms.to_s
+
+        per_group_limit = config.max_snapshots_per_group.to_s
+        groups_limit = config.max_snapshot_groups.to_s
+        bytes = Marshal.dump(page_struct)
+
+        lua = <<~LUA
+          local group_zset_key = KEYS[1]
+          local group_hash_key = KEYS[2]
+          local overview_zset_key = KEYS[3]
+
+          local id = ARGV[1]
+          local score = tonumber(ARGV[2])
+          local group_name = ARGV[3]
+          local per_group_limit = tonumber(ARGV[4])
+          local groups_limit = tonumber(ARGV[5])
+          local prefix = ARGV[6]
+          local bytes = ARGV[7]
+
+          local current_group_score = redis.call("ZSCORE", overview_zset_key, group_name)
+          if current_group_score == false or score > tonumber(current_group_score) then
+            redis.call("ZADD", overview_zset_key, score, group_name)
+          end
+
+          local do_save = true
+          local overview_size = redis.call("ZCARD", overview_zset_key)
+          while (overview_size > groups_limit) do
+            local lowest_group = redis.call("ZRANGE", overview_zset_key, 0, 0)[1]
+            redis.call("ZREM", overview_zset_key, lowest_group)
+            if lowest_group == group_name then
+              do_save = false
+            else
+              local lowest_group_zset_key = prefix .. "-mp-group-snapshot-zset-key-" .. lowest_group
+              local lowest_group_hash_key = prefix .. "-mp-group-snapshot-hash-key-" .. lowest_group
+              redis.call("DEL", lowest_group_zset_key, lowest_group_hash_key)
+            end
+            overview_size = overview_size - 1
+          end
+
+          if do_save then
+            redis.call("ZADD", group_zset_key, score, id)
+            local group_size = redis.call("ZCARD", group_zset_key)
+            while (group_size > per_group_limit) do
+              local lowest_snapshot_id = redis.call("ZRANGE", group_zset_key, 0, 0)[1]
+              redis.call("ZREM", group_zset_key, lowest_snapshot_id)
+              if lowest_snapshot_id == id then
+                do_save = false
+              else
+                redis.call("HDEL", group_hash_key, lowest_snapshot_id)
+              end
+              group_size = group_size - 1
+            end
+            if do_save then
+              redis.call("HSET", group_hash_key, id, bytes)
+            end
+          end
+        LUA
+        redis.eval(
+          lua,
+          keys: [group_zset_key, group_hash_key, overview_zset_key],
+          argv: [id, score, group_name, per_group_limit, groups_limit, @prefix, bytes]
+        )
+      end
+
+      def fetch_snapshots_overview
+        overview_zset_key = snapshot_overview_zset_key
+        groups = redis
+          .zrange(overview_zset_key, 0, -1, withscores: true)
+          .map { |(name, worst_score)| [name, { worst_score: worst_score }] }
+
+        prefixed_group_names = groups.map { |(group_name, _)| group_snapshot_zset_key(group_name) }
+        metadata = redis.eval(<<~LUA, keys: prefixed_group_names)
+          local metadata = {}
+          for i, k in ipairs(KEYS) do
+            local best = redis.call("ZRANGE", k, 0, 0, "WITHSCORES")[2]
+            local count = redis.call("ZCARD", k)
+            metadata[i] = {best, count}
+          end
+          return metadata
+        LUA
+        groups.each.with_index do |(_, hash), index|
+          best, count = metadata[index]
+          hash[:best_score] = best.to_f
+          hash[:snapshots_count] = count.to_i
+        end
+        groups.to_h
+      end
+
+      def fetch_snapshots_group(group_name)
+        group_hash_key = group_snapshot_hash_key(group_name)
+        snapshots = []
+        corrupt_snapshots = []
+        redis.hgetall(group_hash_key).each do |id, bytes|
+          # rubocop:disable Security/MarshalLoad
+          snapshots << Marshal.load(bytes)
+          # rubocop:enable Security/MarshalLoad
+        rescue
+          corrupt_snapshots << id
+        end
+        if corrupt_snapshots.size > 0
+          cleanup_corrupt_snapshots(corrupt_snapshots, group_name)
+        end
+        snapshots
+      end
+
+      def load_snapshot(id, group_name)
+        group_hash_key = group_snapshot_hash_key(group_name)
+        bytes = redis.hget(group_hash_key, id)
+        return if !bytes
+        begin
+          # rubocop:disable Security/MarshalLoad
+          Marshal.load(bytes)
+          # rubocop:enable Security/MarshalLoad
+        rescue
+          cleanup_corrupt_snapshots([id], group_name)
+          nil
+        end
+      end
+
       private
 
       def user_key(user)
@@ -125,6 +276,78 @@ unviewed_ids: #{get_unviewed_ids(user)}
         end
       end
 
+      def snapshot_counter_key
+        @snapshot_counter_key ||= "#{@prefix}-mini-profiler-snapshots-counter"
+      end
+
+      def group_snapshot_zset_key(group_name)
+        # if you change this key, remember to change it in the LUA script in
+        # the push_snapshot method as well
+        "#{@prefix}-mp-group-snapshot-zset-key-#{group_name}"
+      end
+
+      def group_snapshot_hash_key(group_name)
+        # if you change this key, remember to change it in the LUA script in
+        # the push_snapshot method as well
+        "#{@prefix}-mp-group-snapshot-hash-key-#{group_name}"
+      end
+
+      def snapshot_overview_zset_key
+        "#{@prefix}-mp-overviewgroup-snapshot-zset-key"
+      end
+
+      def cached_redis_eval(script, script_sha, reraise: true, argv: [], keys: [])
+        begin
+          redis.evalsha(script_sha, argv: argv, keys: keys)
+        rescue ::Redis::CommandError => e
+          if e.message.start_with?('NOSCRIPT')
+            redis.eval(script, argv: argv, keys: keys)
+          else
+            raise e if reraise
+          end
+        end
+      end
+
+      def cleanup_corrupt_snapshots(corrupt_snapshots_ids, group_name)
+        group_hash_key = group_snapshot_hash_key(group_name)
+        group_zset_key = group_snapshot_zset_key(group_name)
+        overview_zset_key = snapshot_overview_zset_key
+        lua = <<~LUA
+          local group_hash_key = KEYS[1]
+          local group_zset_key = KEYS[2]
+          local overview_zset_key = KEYS[3]
+          local group_name = ARGV[1]
+          for i, k in ipairs(ARGV) do
+            if k ~= group_name then
+              redis.call("HDEL", group_hash_key, k)
+              redis.call("ZREM", group_zset_key, k)
+            end
+          end
+          if redis.call("ZCARD", group_zset_key) == 0 then
+            redis.call("ZREM", overview_zset_key, group_name)
+            redis.call("DEL", group_hash_key, group_zset_key)
+          else
+            local worst_score = tonumber(redis.call("ZRANGE", group_zset_key, -1, -1, "WITHSCORES")[2])
+            redis.call("ZADD", overview_zset_key, worst_score, group_name)
+          end
+        LUA
+        redis.eval(
+          lua,
+          keys: [group_hash_key, group_zset_key, overview_zset_key],
+          argv: [group_name, *corrupt_snapshots_ids]
+        )
+      end
+
+      # only used in tests
+      def wipe_snapshots_data
+        keys = redis.keys(group_snapshot_hash_key('*'))
+        keys += redis.keys(group_snapshot_zset_key('*'))
+        redis.del(
+          keys,
+          snapshot_overview_zset_key,
+          snapshot_counter_key
+        )
+      end
     end
   end
 end

data/lib/mini_profiler/storage.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'mini_profiler/storage/abstract_store'
+require 'mini_profiler/storage/memcache_store'
+require 'mini_profiler/storage/memory_store'
+require 'mini_profiler/storage/redis_store'
+require 'mini_profiler/storage/file_store'

data/lib/mini_profiler/timer_struct/base.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Rack
   class MiniProfiler
     module TimerStruct

data/lib/mini_profiler/timer_struct/custom.rb

@@ -6,6 +6,7 @@ module Rack
       # Timing system for a custom timers such as cache, redis, RPC, external API
       # calls, etc.
       class Custom < TimerStruct::Base
+        attr_accessor :parent
         def initialize(type, duration_ms, page, parent)
           @parent      = parent
           @page        = page

data/lib/mini_profiler/timer_struct/page.rb

@@ -10,6 +10,53 @@ module Rack
       #     :has_many TimerStruct::Sql children
       #     :has_many TimerStruct::Custom children
       class Page < TimerStruct::Base
+        class << self
+          def from_hash(hash)
+            hash = symbolize_hash(hash)
+            if hash.key?(:custom_timing_names)
+              hash[:custom_timing_names] = []
+            end
+            hash.delete(:started_formatted)
+            if hash.key?(:duration_milliseconds)
+              hash[:duration_milliseconds] = 0
+            end
+            page = self.allocate
+            page.instance_variable_set(:@attributes, hash)
+            page
+          end
+
+          private
+
+          def symbolize_hash(hash)
+            new_hash = {}
+            hash.each do |k, v|
+              sym_k = String === k ? k.to_sym : k
+              if Hash === v
+                new_hash[sym_k] = symbolize_hash(v)
+              elsif Array === v
+                new_hash[sym_k] = symbolize_array(v)
+              else
+                new_hash[sym_k] = v
+              end
+            end
+            new_hash
+          end
+
+          def symbolize_array(array)
+            array.map do |item|
+              if Array === item
+                symbolize_array(item)
+              elsif Hash === item
+                symbolize_hash(item)
+              else
+                item
+              end
+            end
+          end
+        end
+
+        attr_reader :attributes
+
         def initialize(env)
           timer_id     = MiniProfiler.generate_id
           page_name    = env['PATH_INFO']
@@ -39,8 +86,13 @@ module Rack
             executed_scalars: 0,
             executed_non_queries: 0,
             custom_timing_names: [],
-            custom_timing_stats: {}
+            custom_timing_stats: {},
+            custom_fields: {},
+            has_flamegraph: false,
+            flamegraph: nil
           )
+          self[:request_method] = env['REQUEST_METHOD']
+          self[:request_path] = env['PATH_INFO']
           name = "#{env['REQUEST_METHOD']} http://#{env['SERVER_NAME']}:#{env['SERVER_PORT']}#{env['SCRIPT_NAME']}#{env['PATH_INFO']}"
           self[:root] = TimerStruct::Request.createRoot(name, self)
         end
@@ -61,17 +113,21 @@ module Rack
           @attributes[:root]
         end
 
+        def attributes_to_serialize
+          @attributes.keys - [:flamegraph]
+        end
+
         def to_json(*a)
-          ::JSON.generate(@attributes.merge(self.extra_json))
+          ::JSON.generate(@attributes.slice(*attributes_to_serialize).merge(extra_json))
         end
 
         def as_json(options = nil)
-          super(options).merge!(extra_json)
+          super(options).slice(*attributes_to_serialize.map(&:to_s)).merge!(extra_json)
         end
 
         def extra_json
           {
-            started: '/Date(%d)/' % @attributes[:started_at],
+            started_formatted: '/Date(%d)/' % @attributes[:started_at],
             duration_milliseconds: @attributes[:root][:duration_milliseconds],
             custom_timing_names: @attributes[:custom_timing_stats].keys.sort
           }

data/lib/mini_profiler/timer_struct/request.rb

@@ -11,7 +11,7 @@ module Rack
           end
         end
 
-        attr_accessor :children_duration
+        attr_accessor :children_duration, :start, :parent
 
         def initialize(name, page, parent)
           start_millis = (Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i - page[:started]
@@ -62,10 +62,6 @@ module Rack
           self[:start_milliseconds]
         end
 
-        def start
-          @start
-        end
-
         def depth
           self[:depth]
         end
@@ -91,6 +87,20 @@ module Rack
           end
         end
 
+        def move_child(child, destination)
+          if index = self[:children].index(child)
+            self[:children].slice!(index)
+            self[:has_children] = self[:children].size > 0
+
+            destination[:children].push(child)
+            destination[:has_children] = true
+
+            child[:parent_timing_id] = destination[:id]
+            child.parent = destination
+            child.adjust_depth
+          end
+        end
+
         def add_sql(query, elapsed_ms, page, params = nil, skip_backtrace = false, full_backtrace = false)
           TimerStruct::Sql.new(query, elapsed_ms, page, self, params, skip_backtrace, full_backtrace).tap do |timer|
             self[:sql_timings].push(timer)
@@ -102,6 +112,19 @@ module Rack
           end
         end
 
+        def move_sql(sql, destination)
+          if index = self[:sql_timings].index(sql)
+            self[:sql_timings].slice!(index)
+            self[:has_sql_timings] = self[:sql_timings].size > 0
+            self[:sql_timings_duration_milliseconds] -= sql[:duration_milliseconds]
+            destination[:sql_timings].push(sql)
+            destination[:has_sql_timings] = true
+            destination[:sql_timings_duration_milliseconds] += sql[:duration_milliseconds]
+            sql[:parent_timing_id] = destination[:id]
+            sql.parent = destination
+          end
+        end
+
         def add_custom(type, elapsed_ms, page)
           TimerStruct::Custom.new(type, elapsed_ms, page, self).tap do |timer|
             timer[:parent_timing_id] = self[:id]
@@ -119,18 +142,37 @@ module Rack
           end
         end
 
+        def move_custom(type, custom, destination)
+          if index = self[:custom_timings][type]&.index(custom)
+            custom[:parent_timing_id] = destination[:id]
+            custom.parent = destination
+            self[:custom_timings][type].slice!(index)
+            if self[:custom_timings][type].size == 0
+              self[:custom_timings].delete(type)
+              self[:custom_timing_stats].delete(type)
+            else
+              self[:custom_timing_stats][type][:count] -= 1
+              self[:custom_timing_stats][type][:duration] -= custom[:duration_milliseconds]
+            end
+            destination[:custom_timings][type] ||= []
+            destination[:custom_timings][type].push(custom)
+            destination[:custom_timing_stats][type] ||= { count: 0, duration: 0.0 }
+            destination[:custom_timing_stats][type][:count] += 1
+            destination[:custom_timing_stats][type][:duration] += custom[:duration_milliseconds]
+          end
+        end
+
         def record_time(milliseconds = nil)
           milliseconds ||= (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start) * 1000
           self[:duration_milliseconds]                  = milliseconds
           self[:is_trivial]                             = true if milliseconds < self[:trivial_duration_threshold_milliseconds]
-          self[:duration_without_children_milliseconds] = milliseconds - @children_duration
-
-          if @parent
-            @parent.children_duration += milliseconds
-          end
-
+          self[:duration_without_children_milliseconds] = milliseconds - self[:children].sum(&:duration_ms)
         end
 
+        def adjust_depth
+          self[:depth] = self.parent ? self.parent[:depth] + 1 : 0
+          self[:children].each(&:adjust_depth)
+        end
       end
     end
   end

data/lib/mini_profiler/timer_struct/sql.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
+require 'erb'
+
 module Rack
   class MiniProfiler
 
     # Timing system for a SQL query
     module TimerStruct
       class Sql < TimerStruct::Base
+        attr_accessor :parent
+
         def initialize(query, duration_ms, page, parent, params = nil, skip_backtrace = false, full_backtrace = false)
 
           stack_trace = nil
@@ -36,12 +40,12 @@ module Rack
           start_millis = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i - page[:started]) - duration_ms
           super(
             execute_type: 3, # TODO
-            formatted_command_string: query,
+            formatted_command_string: query ? ERB::Util.html_escape(query) : nil,
             stack_trace_snippet: stack_trace,
             start_milliseconds: start_millis,
             duration_milliseconds: duration_ms,
             first_fetch_duration_milliseconds: duration_ms,
-            parameters: trim_binds(params),
+            parameters: query ? trim_binds(params) : nil,
             parent_timing_id: nil,
             is_duplicate: false
           )

data/lib/mini_profiler/timer_struct.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'mini_profiler/timer_struct/base'
+require 'mini_profiler/timer_struct/page'
+require 'mini_profiler/timer_struct/sql'
+require 'mini_profiler/timer_struct/custom'
+require 'mini_profiler/timer_struct/client'
+require 'mini_profiler/timer_struct/request'

data/lib/mini_profiler/version.rb

@@ -2,6 +2,7 @@
 
 module Rack
   class MiniProfiler
-    VERSION = '1.0.2'
+    VERSION = '3.1.0'
+    SOURCE_CODE_URI = 'https://github.com/MiniProfiler/rack-mini-profiler'
   end
 end