awesome_explain 0.3.0 → 1.0.0

data/lib/awesome_explain/queue/simple_queue.rb

@@ -0,0 +1,88 @@
+# module AwesomeExplain::Queue
+#   class SimpleQueue
+#     def initialize
+#       @elems = []
+#       @mutex = Mutex.new
+#       @cond_var = ConditionVariable.new
+#     end
+
+#     def <<(elem)
+#       @mutex.synchronize do
+#         @elems << elem
+#         @cond_var.signal
+#       end
+#     end
+
+#     def pop(blocking = true, timeout = nil)
+#       @mutex.synchronize do
+#         if blocking
+#           if timeout.nil?
+#             while @elems.empty?
+#               @cond_var.wait(@mutex)
+#             end
+#           else
+#             timeout_time = Time.now.to_f + timeout
+#             while @elems.empty? && (remaining_time = timeout_time - Time.now.to_f) > 0
+#               @cond_var.wait(@mutex, remaining_time)
+#             end
+#           end
+#         end
+#         raise ThreadError, 'queue empty' if @elems.empty?
+#         sleep 1
+#         @elems.shift
+#       end
+#     end
+#   end
+# end
+
+module AwesomeExplain::Queue
+  class SimpleQueue
+    include Singleton
+
+    def initialize
+      @queue = Queue.new
+      # Thread.new do
+      #   puts 'while true ==============================='
+      #   command = @queue.pop(false)
+      #   # command.run if command
+      # end
+      # @read_io, @write_io = IO.pipe
+    end
+
+    def <<(o)
+      # pop(false) until @queue.size < 2
+      if @queue.size >= 2
+        items = []
+        while @queue.size >= 2
+          items << @queue.pop
+        end
+        Thread.new { sleep 2; puts "Poped #{items}"; puts items.inspect }
+      end
+      puts "Adding to queue @@@@@@@@@@@@@"
+      @queue << o
+      # @write_io << '.'
+      self
+    end
+
+    def pop(nonblock=false)
+      # return unless @queue.size >= 5
+      # @queue.size.times.each do
+
+      # end
+      puts Thread.current.inspect
+      o = @queue.pop(nonblock)
+      # @read_io.read(1)
+      puts "<------ Element poped ------>"
+      puts o
+      o
+    end
+
+    def size
+      @queue.size
+    end
+
+    def to_io
+      @read_io
+    end
+  end
+end

data/lib/awesome_explain/renderers/active_record.rb

@@ -0,0 +1,114 @@
+module AwesomeExplain
+  module Renderers
+    class ActiveRecord
+      attr_reader :result, :query, :sql_explain
+
+      def initialize(query, result = nil)
+        @query = query
+        @result = result || explain_query
+      end
+
+      def explain_query
+        explain = AwesomeExplain::Config.instance.connection.raw_connection.exec(
+          "EXPLAIN (ANALYZE true, COSTS true, FORMAT json) #{query.to_sql}"
+        )
+        explain = explain.map { |h| h.values.first }.join("\n")
+
+        @sql_explain = SqlExplain.new(explain_output: explain)
+      end
+
+      def print
+        table = Terminal::Table.new do |t|
+          general_stats_section t
+          table_stats_section t
+          node_types_section t
+          index_stats_section t
+        end
+        puts table
+      end
+
+      def plan_stats
+        @plan_stats ||= @sql_explain.tree.plan_stats
+      end
+
+      def table_stats
+        @table_stats ||= plan_stats.table_stats
+      end
+
+      def node_type_stats
+        @node_type_stats ||= plan_stats.node_type_stats
+      end
+
+      def index_stats
+        @index_stats ||= plan_stats.index_stats
+      end
+
+      def seq_scans_row
+        color = plan_stats.seq_scans.positive? ? :cyan : :green
+
+        seq_scans_label = AwesomeExplain::Utils::Color.fg_color(
+          color,
+          'Seq Scans'
+        )
+
+        seq_scans_val = AwesomeExplain::Utils::Color.fg_color(
+          color,
+          plan_stats.seq_scans.to_s
+        )
+
+        [seq_scans_label, seq_scans_val]
+      end
+
+      def general_stats_section(t)
+        title = AwesomeExplain::Utils::Color.fg_color :yellow, 'General Stats'
+        t << [{ value: title, alignment: :center, colspan: 2}]
+        t << :separator
+        t << ['Table', 'Count']
+        t << :separator
+        t << ['Total Rows Planned', plan_stats.total_rows_planned]
+        t << ['Total Rows', plan_stats.total_rows]
+        t << ['Total Loops', plan_stats.total_loops]
+        t << seq_scans_row
+        t << ['Indexes Used', plan_stats.index_stats.size]
+      end
+
+      def table_stats_section(t)
+        title = AwesomeExplain::Utils::Color.fg_color :yellow, 'Table Stats'
+        t << :separator
+        t << [{ value: title, alignment: :center, colspan: 2}]
+        t << :separator
+        t << ['Table', 'Count']
+        t << :separator
+        table_stats.each do |table_name, stats|
+          t << [table_name, stats.dig(:count)]
+        end
+      end
+
+      def node_types_section(t)
+        title = AwesomeExplain::Utils::Color.fg_color :yellow, 'Node Type Stats'
+        t << :separator
+        t << [{ value: title, alignment: :center, colspan: 2}]
+        t << :separator
+        t << ['Node Type', 'Count']
+        t << :separator
+        node_type_stats.each do |node_type, stats|
+          t << [node_type, stats.dig(:count)]
+        end
+      end
+
+      def index_stats_section(t)
+        if index_stats.size.positive?
+          title = AwesomeExplain::Utils::Color.fg_color :yellow, 'Index Stats'
+          t << :separator
+          t << [{ value: title, alignment: :center, colspan: 2}]
+          t << :separator
+          t << ['Index Name', 'Count']
+          t << :separator
+          index_stats.each do |index, stats|
+            t << [index, stats.dig(:count)]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/awesome_explain/renderers/base.rb

@@ -0,0 +1,2 @@
+require 'awesome_explain/renderers/mongoid'
+require 'awesome_explain/renderers/active_record'

data/lib/awesome_explain/renderers/mongoid.rb

@@ -3,41 +3,25 @@ module AwesomeExplain
     class Mongoid
       attr_reader :result, :query
 
-      COLOR_ESCAPES = {
-        none: 0, bright: 1, black: 30,
-        red: 31, green: 32, yellow: 33,
-        blue: 34, magenta: 35, cyan: 36,
-        white: 37, default: 39
-      }
-
-      def initialize(query)
+      def initialize(query, result = nil)
         @query = query
+        @result = result || query.explain
       end
 
       def print
-        @result = query.explain
-
-        print_general_info
-      end
-
-      # Text foreground color
-      def fg_color(clr, text = nil)
-        "\x1B[" + (COLOR_ESCAPES[clr] || 0).to_s + 'm' + (text ? text + "\x1B[0m" : '')
-      end
-
-      # Text background color
-      def bg_color(clr, text = nil)
-        "\x1B[" + ((COLOR_ESCAPES[clr] || 0) + 10).to_s + 'm' + (text ?  text + "\x1B[0m" : '')
+        ap result, indent: -2
+        puts
+        puts explain_summary
+        puts
       end
 
-      def print_general_info
-        ap result, indent: -2
+      def explain_summary
         table = Terminal::Table.new do |t|
           winning_plan_label = 'Winning Plan'
           plan_data = winning_plan_data
           winning_plan_str = plan_data[0]
           used_indexes = plan_data[1]
-          winning_plan_label = fg_color :red, winning_plan_label if winning_plan_str =~ /COLLSCAN/
+          winning_plan_label = AwesomeExplain::Utils::Color.fg_color :red, winning_plan_label if winning_plan_str =~ /COLLSCAN/
           t << [winning_plan_label, winning_plan_str]
           t << :separator
           t << ['Used Indexes', used_indexes.join(', ')]
@@ -59,8 +43,8 @@ module AwesomeExplain
             exec_label_ms = 'Execution time(ms)'
 
             if exec_time > 10
-              exec_label = fg_color :red, exec_label
-              exec_label_ms = fg_color :red, exec_label_ms
+              exec_label = AwesomeExplain::Utils::Color.fg_color :red, exec_label
+              exec_label_ms = AwesomeExplain::Utils::Color.fg_color :red, exec_label_ms
             end
             t << [exec_label_ms, exec_time_ms]
             t << :separator
@@ -68,16 +52,18 @@ module AwesomeExplain
           end
         end
 
-        puts
-        puts table
-        puts
+        table
+      end
+
+      def parsed_query
+        root.dig('parsedQuery')
       end
 
       def winning_plan_data
         used_indexes = []
         plan = winning_plan
         plan_str = stage_label_and_stats(plan)
-        plan_str = dig_input_stages(plan.dig('inputStage'), plan_str, used_indexes) if plan['inputStage']
+        plan_str = dig_input_stages(plan.dig('inputStage'), plan_str, used_indexes) if plan&.dig('inputStage')
 
         [plan_str, used_indexes]
       end
@@ -87,15 +73,15 @@ module AwesomeExplain
       end
 
       def winning_plan
-        root.dig('executionStats', 'executionStages') || root.dig('queryPlanner', 'winningPlan')
+        root.dig('executionStats', 'executionStages') || root.dig('queryPlanner', 'winningPlan') || root['stages'].first['$cursor'].dig('queryPlanner', 'winningPlan') || {}
       end
 
       def rejected_plans
-        root.dig('queryPlanner', 'rejectedPlans')
+        root.dig('queryPlanner', 'rejectedPlans') || root['stages']&.first['$cursor'].dig('queryPlanner', 'rejectedPlans') || {}
       end
 
       def execution_stats
-        root.dig('executionStats')
+        root.dig('executionStats') || root['stages']&.first&.dig('$cursor')&.dig('executionStats') || {}
       end
 
       def dig_input_stages(stage, str, used_indexes, input_stages = false)
@@ -123,6 +109,7 @@ module AwesomeExplain
       end
 
       def stage_label_and_stats(stage)
+        return unless stage.present?
         str = "#{stage.dig('stage')} ("
         str += "#{stage.dig('docsExamined')} / " if stage.dig('docsExamined').present?
         str += stage.dig('nReturned').to_s if stage.dig('nReturned').present?

data/lib/awesome_explain/sidekiq_middleware.rb

@@ -0,0 +1,17 @@
+module AwesomeExplain
+  class SidekiqMiddleware
+    # def call(worker_class, job, queue, redis_pool)
+    def call(worker_class, job, queue)
+      begin
+        Thread.current[:sidekiq_worker_class] = worker_class.class.name
+        Thread.current[:sidekiq_job] = job
+        Thread.current[:sidekiq_queue] = queue
+        Thread.current['ae_source'] = 'sidekiq'
+      rescue => exception
+        # Do nothing
+      end
+
+      yield
+    end
+  end
+end

data/lib/awesome_explain/stats/postgresql.rb

@@ -0,0 +1,54 @@
+module AwesomeExplain
+  module Stats
+    class PostgreSQL
+      def self.upsert!
+        upsert_seq_scans!
+        upsert_dml_stats!
+      end
+
+      def self.upsert_seq_scans!
+        result = ActiveRecord::Base.connection.execute(seq_scans_sql).to_a
+        result.each do |row|
+          row = OpenStruct.new(row)
+          pg_seq_scan = AwesomeExplain::PgSeqScan.find_or_create_by({
+            schema_name: row.schema_name,
+            table_name: row.table_name
+          })
+
+          pg_seq_scan.update({
+            seq_scan: row.seq_scan,
+            seq_tup_read: row.seq_tup_read,
+            idx_scan: row.idx_scan,
+            idx_tup_fetch: row.idx_tup_fetch,
+            size_bytes: row.size_bytes
+          })
+        end
+      end
+
+      def self.upsert_dml_stats!
+        result = ActiveRecord::Base.connection.execute(dml_stats_sql).to_a
+        result.each do |row|
+          row = OpenStruct.new(row)
+          pg_dml_stat = AwesomeExplain::PgDmlStat.find_or_create_by({
+            schema_name: row.schema_name,
+            table_name: row.table_name
+          })
+
+          pg_dml_stat.update({
+            total_inserts: row.n_tup_ins,
+            total_updates: row.n_tup_upd,
+            total_deletes: row.n_tup_del,
+          })
+        end
+      end
+
+      def self.seq_scans_sql
+        "select schemaname as schema_name, relname as table_name, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, pg_relation_size(schemaname::text || '.'::text || relname::text) as size_bytes from pg_stat_user_tables"
+      end
+
+      def self.dml_stats_sql
+        "SELECT schemaname as schema_name, relname as table_name, n_tup_ins, n_tup_upd, n_tup_del FROM pg_stat_user_tables"
+      end
+    end
+  end
+end

data/lib/awesome_explain/subscribers/active_record_passive_subscriber.rb

@@ -0,0 +1,82 @@
+module AwesomeExplain::Subscribers
+  class ActiveRecordPassiveSubscriber < ActiveSupport::LogSubscriber
+    def sql(event)
+      if track_sql(event)
+        sql = event.payload[:sql]
+        begin
+          table_name_and_schema = extract_table_name_and_schema(sql)
+          table_name = table_name_and_schema.first
+          schema_name = table_name_and_schema.last
+          request_id = event.payload[:connection_id]
+          binds = event.payload[:binds]
+          cached = event.payload[:name] == 'CACHE'
+          operation = extract_sql_operation(sql)
+          name = event.payload[:name]
+
+          connection_id = event.payload[:connection_id]
+          connection = ::AwesomeExplain::Config.instance.connection
+          explain_uuid = SecureRandom.uuid
+          explain = connection.raw_connection.exec("EXPLAIN (ANALYZE true, COSTS true, FORMAT json) #{sql}")
+          explain = explain.map { |h| h.values.first }.join("\n")
+          explain = ::AwesomeExplain::SqlExplain.new(explain_output: explain)
+          AwesomeExplain::Insights::SqlPlansInsights.add explain.tree.plan_stats
+          AwesomeExplain::Insights::SqlPlansInsights.add_query sql
+        rescue => exception
+          logger.warn sql
+          logger.warn exception.to_s
+          logger.warn exception.backtrace[0..5]
+        end
+      end
+    end
+
+    def track_sql(event)
+      return false unless Thread.current['ae_analyze']
+      return false if event.payload[:connection].class.name == 'ActiveRecord::ConnectionAdapters::SQLite3Adapter'
+      sql = event.payload[:sql]
+      !sql.match(/EXPLAIN|SAVEPOINT|nextval|CREATE|BEGIN|COMMIT|ROLLBACK|begin|commit|rollback|ar_|sql_|pg_|explain|logs|controllers|stacktraces|schema_migrations|delayed_jobs/) &&
+      sql.strip == sql &&
+      event.payload[:name] != 'SCHEMA'
+    end
+
+    def ddm_query?(sql)
+      matched = sql.match(/INSERT|DELETE|UPDATE/)
+      matched.present? && matched[0].present?
+    end
+
+    def resolve_source_name
+      Thread.current['ae_source'] || DEFAULT_SOURCE_NAME
+    end
+
+    def db_logging_enbled?
+      # return true if Thread.current['ae_analyze']
+      return false if Rails.const_defined?('Console')
+      true
+    end
+
+    def extract_sql_operation(sql)
+      sql.match(/SELECT|INSERT|DELETE|UPDATE/)[0]
+    end
+
+    def extract_table_name_and_schema(sql)
+      matched = sql.match(/FROM\s+(\"\w+\")\.?(\"\w+\")?/)
+      return reduce_table_and_schema(matched) if matched && matched[1].present?
+
+      matched = sql.match(/INSERT INTO\s+(\"\w+\")\.?(\"\w+\")?/)
+      return reduce_table_and_schema(matched) if matched && matched[1].present?
+
+      matched = sql.match(/UPDATE\s+(\"\w+\")\.?(\"\w+\")?/)
+      return reduce_table_and_schema(matched) if matched && matched[1].present?
+    end
+
+    def reduce_table_and_schema(matched)
+      if matched[1].present?
+        table_name = matched[2].nil? ? matched[1] : matched[2]
+        table_name = table_name.gsub(/\"/, '')
+        schema_name = matched[2].nil? ? 'public' : matched[1]
+        schema_name = schema_name.gsub(/\"/, '')
+
+        return [table_name, schema_name]
+      end
+    end
+  end
+end