awesome_explain 0.3.0 → 1.0.0

data/lib/awesome_explain/insights/mongoid_insights.rb

@@ -0,0 +1,44 @@
+module AwesomeExplain::Insights
+  class MongoidInsights
+    attr_accessor :command_subscriber, :metrics
+
+    def self.analyze(metrics = [], &block)
+      instance = new
+      instance.init
+      instance.metrics = metrics
+      block_result = instance.instance_eval(&block)
+      instance.tear_down
+      block_result unless metrics.size.positive?
+    end
+
+    def init
+      command_subscribers = Mongoid.default_client.send(:monitoring).subscribers['Command'] || Mongo::Monitoring::Global.subscribers['Command']
+      @command_subscriber = command_subscribers.select do |s|
+        s.class == AwesomeExplain::CommandSubscriber
+      end.first
+      @command_subscriber.clear
+      Thread.current['ae_analyze'] = true
+      Thread.current['ae_source'] = 'console'
+    end
+
+    def tear_down
+      if @command_subscriber.nil?
+        puts 'Configure the command subscriber then try again.'
+        return
+      end
+
+      if @metrics.size.positive?
+        result = {}
+        @metrics.each do |m|
+          result[m] = @command_subscriber.get(m)
+        end
+
+        @command_subscriber.clear
+        return result
+      else
+        @command_subscriber.clear
+      end
+      Thread.current['ae_analyze'] = false
+    end
+  end
+end

data/lib/awesome_explain/insights/sql_plans_insights.rb

@@ -0,0 +1,64 @@
+module AwesomeExplain::Insights
+  class SqlPlansInsights
+    include Singleton
+
+    attr_accessor :plans_stats, :queries
+    attr_reader :mutex
+
+    def initialize
+      @plans_stats = []
+      @queries = []
+      @mutex = Mutex.new
+    end
+
+    def add(plan_stats)
+      with_mutex { @plans_stats << plan_stats }
+    end
+
+    def add_query(query)
+      with_mutex {
+        query = Niceql::Prettifier.prettify_sql query
+        @queries << query
+      }
+    end
+
+    def plans_stats
+      with_mutex { @plans_stats }
+    end
+
+    def queries
+      with_mutex { @queries }
+    end
+
+    def clear
+      plans_stats.clear
+      queries.clear
+    end
+
+    def self.add(plan_stats)
+      instance.add(plan_stats)
+    end
+
+    def self.add_query(query)
+      instance.add_query(query)
+    end
+
+    def self.plans_stats
+      instance.plans_stats
+    end
+
+    def self.queries
+      instance.queries
+    end
+
+    def self.clear
+      instance.clear
+    end
+
+    private
+
+    def with_mutex
+      @mutex.synchronize { yield }
+    end
+  end
+end

data/lib/awesome_explain/kernel.rb

@@ -1,9 +1,21 @@
 module Kernel
   def ae(query)
     return AwesomeExplain::Renderers::Mongoid.new(query).print if mongoid_query?(query)
+    return AwesomeExplain::Renderers::ActiveRecord.new(query).print if active_record_query?(query)
+
     query
   end
 
+  def analyze(&block)
+    ::AwesomeExplain::MongoiddInsights.analyze(&block)
+  end
+
+  def analyze_ar(options = {}, &block)
+    Thread.current['ae_analyze'] = true
+    Thread.current['ae_source'] = 'console'
+    ::AwesomeExplain::Insights::ActiveRecordInsights.analyze(options, &block)
+  end
+
   private
 
   def mongoid_query?(query)
@@ -11,4 +23,9 @@ module Kernel
       defined?(Mongoid::Criteria) &&
       (query.is_a?(Mongo::Collection::View::Aggregation) || query.is_a?(Mongoid::Criteria))
   end
+
+  def active_record_query?(query)
+    defined?(ActiveRecord::Relation) &&
+      query.is_a?(ActiveRecord::Relation)
+  end
 end

data/lib/awesome_explain/mongodb/base.rb

@@ -0,0 +1,4 @@
+require 'awesome_explain/mongodb/helpers'
+require 'awesome_explain/mongodb/formatter'
+require 'awesome_explain/mongodb/command_start'
+require 'awesome_explain/mongodb/command_success'

data/lib/awesome_explain/mongodb/command_start.rb

@@ -0,0 +1,84 @@
+module AwesomeExplain::Mongodb
+  module CommandStart
+    extend ActiveSupport::Concern
+
+    included do
+      def handle_command_start(event)
+        command = event.command
+        command_name = event.command_name.to_sym
+
+        if db_explain_enabled?(command_name)
+          case AwesomeExplain::Config.instance.enabled?
+          when Rails.env.development?
+            handle_command_start_sync(event)
+          when Rails.env.staging?
+            handle_command_start_async(event)
+          end
+        end
+      end
+
+      def process_command_start(event)
+        command = event.command
+        command_name = event.command_name.to_sym
+
+        begin
+          request_id = event.request_id
+          if command_name == :getMore
+            collection_name = event.command['collection']
+          else
+            collection_name = event.command[event.command_name]
+          end
+          @stats[:collections][collection_name] = Hash.new(0) if !@stats[:collections].include?(collection_name)
+          @stats[:collections][collection_name][command_name] += 1
+          @queries[request_id] = {
+            command_name: event.command_name,
+            command: command.include?('pipeline') ? command['pipeline'] : command.select {|k, v| COMMAND_ALLOWED_KEYS.include?(k)},
+            collection_name: collection_name,
+            stacktrace: caller,
+            lsid: command.dig('lsid', 'id').to_json
+          }.with_indifferent_access
+
+          command = event.command
+          if command.include?('aggregate')
+            command = {
+              'aggregate': command['aggregate'],
+              'pipeline': command['pipeline'],
+              'cursor': command['cursor'],
+            }
+          end
+          r = ::AwesomeExplain::Renderers::Mongoid.new(nil, Mongoid.default_client.database.command({explain: command}).documents.first)
+          exp = AwesomeExplain::Explain.create({
+            collection: collection_name,
+            source_name: AwesomeExplain::Config.instance.app_name,
+            command: @queries[request_id][:command].to_json,
+            winning_plan: r.winning_plan_data.first,
+            winning_plan_raw: r.winning_plan.to_json,
+            used_indexes: r.winning_plan_data.last.join(', '),
+            duration: (r.execution_stats&.dig('executionTimeMillis').to_f/1000).round(5),
+            documents_returned: r.execution_stats&.dig('nReturned'),
+            documents_examined: r.execution_stats&.dig('totalDocsExamined'),
+            keys_examined: r.execution_stats&.dig('totalKeysExamined'),
+            rejected_plans: r.rejected_plans&.size,
+            session_id: Thread.current[:ae_session_id],
+            lsid: @queries[request_id][:lsid],
+            stacktrace_id: resolve_stracktrace_id(request_id),
+            controller_id: resolve_controller_id,
+          })
+          @queries[request_id][:explain_id] = exp&.id
+          @queries[request_id][:collscan] = exp&.collscan
+        rescue => exception
+          logger.warn exception.to_s
+          logger.warn exception.backtrace[0..5]
+        end
+      end
+
+      def handle_command_start_sync(event)
+        # process_command_start(event)
+        ::AwesomeExplain::Config.instance.queue << ::AwesomeExplain::Queue::Command.new(:process_command_start, event, self)
+      end
+
+      def handle_command_start_async(event)
+      end
+    end
+  end
+end

data/lib/awesome_explain/mongodb/command_success.rb

@@ -0,0 +1,58 @@
+module AwesomeExplain::Mongodb
+  module CommandSuccess
+    extend ActiveSupport::Concern
+
+    included do
+      def handle_command_success(event)
+        if db_logging_enbled?
+          case AwesomeExplain::Config.instance.enabled?
+          when Rails.env.development?
+            handle_command_success_sync(event)
+          when Rails.env.staging?
+            handle_command_success_async(event)
+          end
+        end
+      end
+
+      def process_command_success(event)
+        begin
+          command_name = event.command_name.to_sym
+          request_id = event.request_id
+          duration = event.duration.round(5)
+          @stats[:performed_queries][command_name] += 1
+          @stats[:total_duration] += duration
+          @queries[request_id][:duration] = duration
+
+          log = {
+            operation: command_name,
+            app_name: AwesomeExplain::Config.instance.app_name,
+            source_name: resolve_source_name,
+            collscan: @queries[request_id][:collscan],
+            collection: @queries[request_id][:collection_name],
+            duration: duration,
+            command: @queries[request_id][:command].to_json,
+            session_id: Thread.current[:ae_session_id],
+            lsid: @queries[request_id][:lsid],
+            stacktrace_id: resolve_stracktrace_id(request_id),
+            explain_id: @queries[request_id][:explain_id],
+            controller_id: resolve_controller_id,
+            sidekiq_worker_id: resolve_sidekiq_class_id,
+          }
+          AwesomeExplain::Log.create(log)
+        rescue => exception
+          logger.warn exception.to_s
+          logger.warn exception.backtrace[0..5]
+        end
+      end
+
+      def handle_command_success_sync(event)
+        # process_command_success(event)
+        ::AwesomeExplain::Config.instance.queue << ::AwesomeExplain::Queue::Command.new(:process_command_success, event, self)
+      end
+
+      def handle_command_success_async(event)
+        # TODO: How to handle using Queues
+      end
+    end
+  end
+end

data/lib/awesome_explain/mongodb/formatter.rb

@@ -0,0 +1,62 @@
+module AwesomeExplain::Mongodb
+  module Formatter
+    extend ActiveSupport::Concern
+
+    included do
+      def stats_table
+        table = Terminal::Table.new(title: 'Query Stats') do |t|
+          t << [
+            'Total Duration',
+            @stats[:total_duration] >= 1 ? "#{@stats[:total_duration]} seconds".purpleish : "#{@stats[:total_duration]} seconds".green
+          ]
+          t << :separator
+          t << [
+            total_performed_queries >= 100 ? "Performed Queries [#{total_performed_queries}]".purpleish : "Performed Queries [#{total_performed_queries}]".green,
+            formatted_performed_queries
+          ]
+          t << :separator
+          t << ['Collections Queried', formatted_collections]
+
+          sq = slowest_query
+          if sq[:duration] >= 0.5
+            t << :separator
+            t << ["Slowest Query [#{sq.dig(:duration)}]".purpleish, sq.dig(:command).inspect]
+          end
+        end
+        puts table
+      end
+
+      def slowest_query
+        @queries.sort_by {|id, data| data[:duration]}.last[1]
+      end
+
+      def total_performed_queries
+        @stats[:performed_queries].sum {|op, count| count}
+      end
+
+      def formatted_performed_queries
+        find = @stats[:performed_queries][:find]
+        count = @stats[:performed_queries][:count]
+        distinct = @stats[:performed_queries][:distinct]
+        update = @stats[:performed_queries][:update]
+        insert = @stats[:performed_queries][:insert]
+        get_more = @stats[:performed_queries][:getMore]
+        delete = @stats[:performed_queries][:delete]
+
+        QUERIES.reject { |q| !@stats[:performed_queries][q].positive? }.sort_by {|q| @stats[:performed_queries][q]}.reverse.map do |q|
+          @stats[:performed_queries][q] >= 10 ? "#{q}: #{@stats[:performed_queries][q]}".purpleish : "#{q}: #{@stats[:performed_queries][q]}"
+        end.join(', ')
+      end
+
+      def formatted_collections
+        @stats[:collections].sort_by {|c, values| values.sum {|k, v| v}}.reverse.each.map do |data|
+          c = data[0]
+          sum = @stats[:collections][c].sum {|k, v| v}
+          cmds = @stats[:collections][c].map {|cmd, count| "#{cmd} [#{count}]"}.join(', ')
+          cmds = "#{c}: #{cmds}"
+          sum >= 10 ? cmds.purpleish : cmds
+        end.join("\n")
+      end
+    end
+  end
+end

data/lib/awesome_explain/mongodb/helpers.rb

@@ -0,0 +1,71 @@
+module AwesomeExplain::Mongodb
+  module Helpers
+    extend ActiveSupport::Concern
+
+    included do
+      def resolve_stracktrace_id(request_id)
+        stacktrace_str = @queries[request_id][:stacktrace]
+          .select {|c| c =~ /^#{Rails.root.to_s + '\/(lib|app|db)\/'}/ }
+          .map {|c| c.gsub Rails.root.to_s, ''}
+          .to_json
+        stacktrace = AwesomeExplain::Stacktrace.find_or_create_by({
+          stacktrace: stacktrace_str
+        })
+
+        stacktrace.id
+      end
+
+      def resolve_controller_id
+        data = controller_data
+        return nil unless data.present?
+        AwesomeExplain::Controller.find_or_create_by({
+          name: controller_data[:controller],
+          action: controller_data[:action],
+          path: controller_data[:path],
+          params: controller_data[:params].to_json,
+          session_id: Thread.current['ae_session_id']
+        }).id
+      end
+
+      def resolve_sidekiq_class_id
+        return unless Thread.current[:sidekiq_worker_class].present?
+        sidekiq_worker_class_str = Thread.current[:sidekiq_worker_class]
+        sidekiq_queue_str = Thread.current[:sidekiq_queue].to_s
+        sidekiq_worker = AwesomeExplain::SidekiqWorker.find_or_create_by({
+          worker: sidekiq_worker_class_str,
+          queue: sidekiq_queue_str,
+          jid: extract_sidekiq_jid(Thread.current[:sidekiq_job]),
+          params: Thread.current[:sidekiq_job].present? ? Thread.current[:sidekiq_job].to_json : {}
+        })
+
+        sidekiq_worker.id
+      end
+
+      def controller_data
+        Thread.current['ae_controller_data']
+      end
+
+      def extract_sidekiq_jid(args)
+        Thread.current[:sidekiq_job].dig('jid')
+      end
+
+      def resolve_source_name
+        Thread.current['ae_source'] || DEFAULT_SOURCE_NAME
+      end
+
+      def db_explain_enabled?(command_name)
+        return false if DML_COMMANDS.include?(command_name)
+        return false if command_name == :getMore
+        return true if Thread.current['ae_analyze']
+        return false if Rails.const_defined?('Console')
+        true
+      end
+
+      def db_logging_enbled?
+        return true if Thread.current['ae_analyze']
+        return false if Rails.const_defined?('Console')
+        true
+      end
+    end
+  end
+end

data/lib/awesome_explain/queue/command.rb

@@ -0,0 +1,17 @@
+module AwesomeExplain::Queue
+  class Command
+    attr_writer :method_name
+    attr_writer :event
+    attr_writer :object
+
+    def initialize(method_name, event, object)
+      @method_name = method_name
+      @event = event
+      @object = object
+    end
+
+    def run
+      object.send method_name, event
+    end
+  end
+end