awesome_explain 0.3.0 → 1.0.0

data/lib/awesome_explain/subscribers/active_record_subscriber.rb

@@ -0,0 +1,187 @@
+module AwesomeExplain::Subscribers
+  class ActiveRecordSubscriber < ActiveSupport::LogSubscriber
+    def sql(event)
+      if track_sql(event) && db_logging_enbled?
+        sql = event.payload[:sql]
+        begin
+          table_name_and_schema = extract_table_name_and_schema(sql)
+          table_name = table_name_and_schema.first
+          schema_name = table_name_and_schema.last
+          request_id = event.payload[:connection_id]
+          binds = event.payload[:binds]
+          cached = event.payload[:name] == 'CACHE'
+          operation = extract_sql_operation(sql)
+          name = event.payload[:name]
+          stacktrace = caller
+
+          ActiveRecord::Base.transaction do
+            explain = nil
+            if db_explain_enabled?(sql)# && !cached
+              connection_id = event.payload[:connection_id]
+              connection = ::AwesomeExplain::Config.instance.connection
+
+              explain_uuid = SecureRandom.uuid
+              explain = connection.raw_connection.exec("EXPLAIN (ANALYZE true, COSTS true, FORMAT json) #{sql}")
+              #explain = connection.raw_connection.exec_prepared("ae_#{explain_uuid}", binds).to_a
+              explain = explain.map { |h| h.values.first }.join("\n")
+
+              explain = ::AwesomeExplain::SqlExplain.new({
+                explain_output: explain,
+                stacktrace_id: resolve_stracktrace_id(stacktrace),
+                controller_id: resolve_controller_id,
+              })
+              explain.save
+            end
+
+            sql_query = {
+              table_name: table_name,
+              schema_name: schema_name,
+              app_name: ::AwesomeExplain::Config.instance.app_name,
+              source_name: resolve_source_name,
+              duration: event.duration,
+              query: sql,
+              binds: binds.to_json,
+              cached: cached,
+              name: name,
+              operation: operation,
+              session_id: Thread.current[:ae_session_id],
+              stacktrace_id: resolve_stracktrace_id(stacktrace),
+              sql_explain_id: explain&.id,
+              controller_id: resolve_controller_id,
+              sidekiq_worker_id: resolve_sidekiq_class_id,
+              delayed_job_id: resolve_delayed_job_class_id,
+            }
+            ::AwesomeExplain::SqlQuery.create(sql_query)
+          end
+        rescue => exception
+          logger.warn sql
+          logger.warn exception.to_s
+          logger.warn exception.backtrace[0..5]
+        end
+      end
+    end
+
+    def track_sql(event)
+      return false if event.payload[:connection].class.name == 'ActiveRecord::ConnectionAdapters::SQLite3Adapter'
+      sql = event.payload[:sql]
+      !sql.match(/EXPLAIN|SAVEPOINT|nextval|CREATE|BEGIN|COMMIT|ROLLBACK|begin|commit|rollback|ar_|sql_|pg_|explain|logs|controllers|stacktraces|schema_migrations|delayed_jobs/) &&
+      sql.strip == sql &&
+      event.payload[:name] != 'SCHEMA'
+    end
+
+    def ddm_query?(sql)
+      matched = sql.match(/INSERT|DELETE|UPDATE/)
+      matched.present? && matched[0].present?
+    end
+
+    def resolve_source_name
+      Thread.current['ae_source'] || DEFAULT_SOURCE_NAME
+    end
+
+    def controller_data
+      Thread.current['ae_controller_data']
+    end
+
+    def extract_sidekiq_jid(args)
+      Thread.current[:sidekiq_job].dig('jid')
+    end
+
+    def extract_delayed_job_jid(args)
+      Thread.current[:delayed_job]&.id
+    end
+
+    def resolve_stracktrace_id(stacktrace)
+      stacktrace_str = stacktrace
+        .select {|c| c =~ /^#{::AwesomeExplain::Config.instance.rails_path + '\/(lib|app|db)\/'}/ }
+        .map {|c| c.gsub Rails.root.to_s, ''}
+        .to_json
+
+      stacktrace = ::AwesomeExplain::Stacktrace.find_or_create_by({
+        stacktrace: stacktrace_str
+      })
+
+      stacktrace.id
+    end
+
+    def resolve_controller_id
+      data = controller_data
+      return nil unless data.present?
+      ::AwesomeExplain::Controller.find_or_create_by({
+        name: controller_data[:controller],
+        action: controller_data[:action],
+        path: controller_data[:path],
+        params: controller_data[:params].to_json,
+        session_id: Thread.current['ae_session_id']
+      }).id
+    end
+
+    def resolve_sidekiq_class_id
+      return unless Thread.current[:sidekiq_worker_class].present?
+      sidekiq_worker_class_str = Thread.current[:sidekiq_worker_class]
+      sidekiq_queue_str = Thread.current[:sidekiq_queue].to_s
+      sidekiq_worker = ::AwesomeExplain::SidekiqWorker.find_or_create_by({
+        worker: sidekiq_worker_class_str,
+        queue: sidekiq_queue_str,
+        jid: extract_sidekiq_jid(Thread.current[:sidekiq_job]),
+        params: Thread.current[:sidekiq_job].present? ? Thread.current[:sidekiq_job].to_json : {}
+      })
+
+      sidekiq_worker.id
+    end
+
+    def resolve_delayed_job_class_id
+      return unless Thread.current[:delayed_worker_class].present?
+      delayed_worker_class_str = Thread.current[:delayed_worker_class]
+      delayed_job_queue_str = Thread.current[:delayed_job_queue].to_s
+      delayed_job_worker = ::AwesomeExplain::DelayedJob.find_or_create_by({
+        job: delayed_worker_class_str,
+        jid: extract_delayed_job_jid(Thread.current[:delayed_job]),
+        params: Thread.current[:delayed_job].present? ? Thread.current[:delayed_job].to_json : {}
+      })
+
+      delayed_job_worker.id
+    end
+
+    def db_explain_enabled?(sql)
+      # return true if Thread.current['ae_analyze']
+      # return false if Rails.const_defined?('Console')
+      return false if ddm_query?(sql)
+      true
+    end
+
+    def db_logging_enbled?
+      # return true if Thread.current['ae_analyze']
+      return false if Rails.const_defined?('Console')
+      true
+    end
+
+    def extract_sql_operation(sql)
+      sql.match(/SELECT|INSERT|DELETE|UPDATE/)[0]
+    end
+
+    def extract_table_name_and_schema(sql)
+      matched = sql.match(/FROM\s+(\"\w+\")\.?(\"\w+\")?/)
+      return reduce_table_and_schema(matched) if matched && matched[1].present?
+
+      matched = sql.match(/INSERT INTO\s+(\"\w+\")\.?(\"\w+\")?/)
+      return reduce_table_and_schema(matched) if matched && matched[1].present?
+
+      matched = sql.match(/UPDATE\s+(\"\w+\")\.?(\"\w+\")?/)
+      return reduce_table_and_schema(matched) if matched && matched[1].present?
+
+      # matched = sql.match(/DELETE FROM\s+(\"\w+\")\.?(\"\w+\")?/)
+      # return reduce_table_and_schema(matched) if matched && matched[1].present?
+    end
+
+    def reduce_table_and_schema(matched)
+      if matched[1].present?
+        table_name = matched[2].nil? ? matched[1] : matched[2]
+        table_name = table_name.gsub(/\"/, '')
+        schema_name = matched[2].nil? ? 'public' : matched[1]
+        schema_name = schema_name.gsub(/\"/, '')
+
+        return [table_name, schema_name]
+      end
+    end
+  end
+end

data/lib/awesome_explain/subscribers/base.rb

@@ -0,0 +1,3 @@
+require 'awesome_explain/subscribers/command_subscriber'
+require 'awesome_explain/subscribers/active_record_subscriber'
+require 'awesome_explain/subscribers/active_record_passive_subscriber'

data/lib/awesome_explain/subscribers/command_subscriber.rb

@@ -0,0 +1,53 @@
+module AwesomeExplain::Subscribers
+  class CommandSubscriber
+    include AwesomeExplain::Mongodb::Formatter
+    include AwesomeExplain::Mongodb::Helpers
+    include AwesomeExplain::Mongodb::CommandStart
+    include AwesomeExplain::Mongodb::CommandSuccess
+
+    attr_reader :logger
+    attr_accessor :options, :queries, :stats
+
+    def initialize(options = {})
+      init(options)
+      @logger = ::AwesomeExplain::Config.instance.logger
+    end
+
+    def started(event)
+      unless COMMAND_NAMES_BLACKLIST.include?(event.command_name)
+        handle_command_start(event)
+      end
+    end
+
+    def succeeded(event)
+      unless COMMAND_NAMES_BLACKLIST.include?(event.command_name)
+        handle_command_success(event)
+      end
+    end
+
+    def failed(event)
+    end
+
+    def get(metric)
+      case metric
+      when :total_performed_queries
+        total_performed_queries
+      end
+    end
+
+    def clear
+      init
+    end
+
+    private
+    def init(options = {})
+      @options = options
+      @queries = Hash.new({})
+      @stats = {
+        total_duration: 0,
+        collections: {},
+        performed_queries: QUERIES.inject(Hash.new) {|h, q| h[q] = 0; h}
+      }
+    end
+  end
+end

data/lib/awesome_explain/tasks/db.rb

@@ -0,0 +1,325 @@
+require 'pg'
+
+module AwesomeExplain
+  module Tasks
+    class DB
+      def self.build
+        ActiveRecord::Base.establish_connection(
+          AwesomeExplain::Config.instance.db_config
+        )
+
+        adapter = AwesomeExplain::Config.instance.adapter&.to_sym
+        adapter == :postgres ? build_postgres_db : build_sqlite3_db
+      end
+
+      def self.drop_postgres_db
+        conn = PG.connect(
+          dbname: 'postgres',
+          host: postgres_host,
+          user: postgres_username,
+          password: postgres_password
+        )
+
+        conn.exec("DROP DATABASE #{AwesomeExplain::Config::POSTGRES_DEV_DBNAME}")
+      end
+
+      def self.build_postgres_db
+        conn = PG.connect(
+          dbname: 'postgres',
+          host: postgres_host,
+          user: postgres_username,
+          password: postgres_password
+        )
+        sql = <<-SQL
+          select exists(
+            SELECT datname FROM pg_catalog.pg_database WHERE lower(datname) = lower('#{AwesomeExplain::Config::POSTGRES_DEV_DBNAME}')
+          );
+        SQL
+
+        if conn.exec(sql).to_a.first.dig('exists') == 'f'
+          conn.exec("CREATE DATABASE #{AwesomeExplain::Config::POSTGRES_DEV_DBNAME}")
+        end
+
+        connection = ActiveRecord::Base.establish_connection(
+          AwesomeExplain::Config.instance.db_config
+        ).connection
+
+        ActiveRecord::Schema.define do
+          unless connection.table_exists?(:stacktraces)
+            connection.create_table :stacktraces do |t|
+              t.column :stacktrace, :string
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:sidekiq_workers)
+            connection.create_table :sidekiq_workers do |t|
+              t.column :worker, :string
+              t.column :queue, :string
+              t.column :jid, :string
+              t.column :params, :string
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:delayed_jobs)
+            connection.create_table :delayed_jobs do |t|
+              t.column :job, :string
+              t.column :queue, :string
+              t.column :jid, :string
+              t.column :params, :string
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:controllers)
+            connection.create_table :controllers do |t|
+              t.column :name, :string
+              t.column :action, :string
+              t.column :path, :string
+              t.column :params, :string
+              t.column :session_id, :string
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:logs)
+            connection.create_table :logs do |t|
+              t.column :collection, :string
+              t.column :app_name, :string
+              t.column :source_name, :string
+              t.column :operation, :string
+              t.column :collscan, :integer
+              t.column :command, :string
+              t.decimal :duration, precision: 8, scale: 2
+              t.column :session_id, :string
+              t.column :lsid, :string
+
+              t.column :sidekiq_args, :string
+              t.column :stacktrace_id, :integer
+              t.column :explain_id, :integer
+              t.column :controller_id, :integer
+              t.column :sidekiq_worker_id, :integer
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:sql_queries)
+            connection.create_table :sql_queries do |t|
+              t.column :table_name, :string
+              t.column :schema_name, :string
+              t.column :app_name, :string
+              t.column :source_name, :string
+              t.column :operation, :string
+              t.column :query, :string
+              t.column :binds, :string
+              t.decimal :duration, precision: 8, scale: 2
+              t.column :session_id, :string
+              t.column :cached, :integer
+              t.column :name, :string
+
+              t.column :sidekiq_args, :string
+              t.column :stacktrace_id, :integer
+              t.column :sql_explain_id, :integer
+              t.column :controller_id, :integer
+              t.column :sidekiq_worker_id, :integer
+              t.column :delayed_job_id, :integer
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:explains)
+            connection.create_table :explains do |t|
+              t.column :collection, :string
+              t.column :source_name, :string
+              t.column :command, :string
+              t.column :collscan, :integer
+              t.column :winning_plan, :string
+              t.column :winning_plan_raw, :string
+              t.column :used_indexes, :string
+              t.decimal :duration, precision: 8, scale: 2
+              t.column :documents_returned, :integer
+              t.column :documents_examined, :integer
+              t.column :keys_examined, :integer
+              t.column :rejected_plans, :integer
+              t.column :session_id, :string
+              t.column :lsid, :string
+              t.column :stacktrace_id, :integer
+              t.column :controller_id, :integer
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:sql_explains)
+            connection.create_table :sql_explains do |t|
+              t.column :explain_output, :string
+              t.column :stacktrace_id, :integer
+              t.column :controller_id, :integer
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:transactions)
+            connection.create_table :transactions do |t|
+              t.column :params, :string
+              t.column :format, :string
+              t.column :method, :string
+              t.column :ip, :string
+              t.column :stash, :string
+              t.column :status, :string
+              t.column :view_runtime, :string
+
+              t.column :controller_id, :integer
+              t.timestamps
+            end
+          end
+        end
+      end
+
+      def self.build_sqlite3_db
+        ActiveRecord::Schema.define do
+          unless connection.table_exists?(:stacktraces)
+            connection.create_table :stacktraces do |t|
+              t.column :stacktrace, :string
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:sidekiq_workers)
+            connection.create_table :sidekiq_workers do |t|
+              t.column :worker, :string
+              t.column :queue, :string
+              t.column :jid, :string
+              t.column :params, :string
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:delayed_jobs)
+            connection.create_table :delayed_jobs do |t|
+              t.column :job, :string
+              t.column :queue, :string
+              t.column :jid, :string
+              t.column :params, :string
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:controllers)
+            connection.create_table :controllers do |t|
+              t.column :name, :string
+              t.column :action, :string
+              t.column :path, :string
+              t.column :params, :string
+              t.column :session_id, :string
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:logs)
+            connection.create_table :logs do |t|
+              t.column :collection, :string
+              t.column :app_name, :string
+              t.column :source_name, :string
+              t.column :operation, :string
+              t.column :collscan, :integer
+              t.column :command, :string
+              t.column :duration, :double
+              t.column :session_id, :string
+              t.column :lsid, :string
+
+              t.column :sidekiq_args, :string
+              t.column :stacktrace_id, :integer
+              t.column :explain_id, :integer
+              t.column :controller_id, :integer
+              t.column :sidekiq_worker_id, :integer
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:sql_queries)
+            connection.create_table :sql_queries do |t|
+              t.column :table_name, :string
+              t.column :schema_name, :string
+              t.column :app_name, :string
+              t.column :source_name, :string
+              t.column :operation, :string
+              t.column :query, :string
+              t.column :binds, :string
+              t.column :duration, :double
+              t.column :session_id, :string
+              t.column :cached, :integer
+              t.column :name, :string
+
+              t.column :sidekiq_args, :string
+              t.column :stacktrace_id, :integer
+              t.column :sql_explain_id, :integer
+              t.column :controller_id, :integer
+              t.column :sidekiq_worker_id, :integer
+              t.column :delayed_job_id, :integer
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:explains)
+            connection.create_table :explains do |t|
+              t.column :collection, :string
+              t.column :source_name, :string
+              t.column :command, :string
+              t.column :collscan, :integer
+              t.column :winning_plan, :string
+              t.column :winning_plan_raw, :string
+              t.column :used_indexes, :string
+              t.column :duration, :double
+              t.column :documents_returned, :integer
+              t.column :documents_examined, :integer
+              t.column :keys_examined, :integer
+              t.column :rejected_plans, :integer
+              t.column :session_id, :string
+              t.column :lsid, :string
+              t.column :stacktrace_id, :integer
+              t.column :controller_id, :integer
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:sql_explains)
+            connection.create_table :sql_explains do |t|
+              t.column :explain_output, :string
+              t.column :stacktrace_id, :integer
+              t.column :controller_id, :integer
+              t.timestamps
+            end
+          end
+
+          unless connection.table_exists?(:transactions)
+            connection.create_table :transactions do |t|
+              t.column :params, :string
+              t.column :format, :string
+              t.column :method, :string
+              t.column :ip, :string
+              t.column :stash, :string
+              t.column :status, :string
+              t.column :view_runtime, :string
+
+              t.column :controller_id, :integer
+              t.timestamps
+            end
+          end
+        end
+      end
+
+      def self.postgres_host
+        AwesomeExplain::Config.instance.postgres_host
+      end
+
+      def self.postgres_username
+        AwesomeExplain::Config.instance.postgres_username
+      end
+
+      def self.postgres_password
+        AwesomeExplain::Config.instance.postgres_password
+      end
+    end
+  end
+end