query_police 0.1.0.beta

data/lib/query_police/analysis/dynamic_message.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module QueryPolice
+  class Analysis
+    # Module to define methods related to dynamic message
+    module DynamicMessage
+      private
+
+      LISTED_VAR = %w[amount column impact table tag value].freeze
+
+      # to pretty print the analysis with warnings and suggestions
+      # @param opts [Hash] opts to get specifc dyanmic message
+      # eg. {"table" => "users", "column" => "select_type", "tag" => "SIMPLE", "type" => "message"}
+      # @return [String]
+      def dynamic_message(opts)
+        table, column, tag, type = opts.values_at("table", "column", "tag", "type")
+        message = self.tables.dig(table, "analysis", column, "tags", tag, type) || ""
+
+        variables = message.scan(/\$(\w+)/).uniq.map { |var| var[0] }
+        variables.each do |var|
+          value = dynamic_value_of(var, opts)
+
+          message.gsub!(/\$#{var}/, value.to_s) if value.present?
+        end
+
+        message
+      end
+
+      def dynamic_value_of(var, opts)
+        LISTED_VAR.include?(var) ? send(var, opts) : relative_value_of(var, opts.dig("table"))
+      end
+
+      def relative_value_of(var, table)
+        value_type = var.match(/amount_/).present? ? "amount" : "value"
+        self.tables.dig(table, "analysis", var.gsub(/amount_/, ""), value_type)
+      end
+
+      # dynamic variable methods
+      def amount(opts)
+        table, column = opts.values_at("table", "column")
+
+        self.tables.dig(table, "analysis", column, "amount")
+      end
+
+      def column(opts)
+        opts.dig("column")
+      end
+
+      def impact(opts)
+        table, column, tag = opts.values_at("table", "column", "tag")
+
+        impact = self.tables.dig(table, "analysis", column, "tags", tag, "impact")
+
+        opts.dig("colours").present? ? impact.send(IMPACTS[impact].colour) : impact
+      end
+
+      def table(opts)
+        opts.dig("table")
+      end
+
+      def tag(opts)
+        opts.dig("tag")
+      end
+
+      def value(opts)
+        table, column = opts.values_at("table", "column")
+
+        self.tables.dig(table, "analysis", column, "value")
+      end
+    end
+  end
+end

data/lib/query_police/analysis.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require_relative 'analysis/dynamic_message'
+
+module QueryPolice
+  # This class is used to store analysis of a query and provide methods over them
+  class Analysis
+    include DynamicMessage
+
+    IMPACTS = {
+      "negative" => { "colour" => "red" },
+      "positive" => { "colour" => "green" },
+      "caution" => { "colour" => "yellow" }
+    }.freeze
+
+    # initialize analysis object
+    # tables [Array] Array of table analysis
+    # Eg.
+    # {
+    #   "users" => {
+    #     "id"=>1,
+    #     "name"=>"users",
+    #     "analysis"=>{
+    #       "type"=>{
+    #         "value" => "all",
+    #         "tags" => {
+    #           "all" => {
+    #             "impact"=>"negative",
+    #             "warning"=>"warning to represent the issue",
+    #             "suggestions"=>"some follow up suggestions"
+    #           }
+    #         }
+    #       }
+    #     }
+    #   }
+    # }
+    # summary [Hash] hash of analysis summary
+    # Eg.
+    #  {
+    #    "cardinality"=>{
+    #      "amount"=>10,
+    #      "warning"=>"warning to represent the issue",
+    #      "suggestions"=>"some follow up suggestions"
+    #    }
+    #  }
+    def initialize
+      @table_count = 0
+      @tables = {}
+      @summary = {}
+    end
+
+    attr_accessor :tables, :table_count, :summary
+
+    # register a table analysis in analysis object
+    # @param name [String] name of the table
+    # @param table_analysis [Hash] analysis of a table
+    # Eg.
+    #  {
+    #    "id"=>1,
+    #    "name"=>"users",
+    #    "analysis"=>{
+    #      "type"=>[
+    #        {
+    #          "tag"=>"all",
+    #          "impact"=>"negative",
+    #          "warning"=>"warning to represent the issue",
+    #          "suggestions"=>"some follow up suggestions"
+    #        }
+    #      ]
+    #    }
+    #  }
+    def register_table(name, table_analysis)
+      self.table_count += 1
+      self.tables.merge!(
+        {
+          name => {
+            "id" => self.table_count,
+            "name" => name,
+            "analysis" => table_analysis
+          }
+        }
+      )
+    end
+
+    # register summary based in different attributes
+    # @param summary [Hash] hash of summary of analysis
+    def register_summary(summary)
+      self.summary.merge!(summary)
+    end
+
+    # to get analysis in pretty format with warnings and suggestions
+    # @param opts [Hash] - possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
+    # @return [String] pretty analysis 
+    def pretty_analysis(opts)
+      final_message = ""
+
+      opts.slice(*IMPACTS.keys).each do |impact, value|
+        final_message += pretty_analysis_for(impact) if value.present?
+      end
+
+      final_message
+    end
+
+    # to get analysis in pretty format with warnings and suggestions for a impact
+    # @param impact [String]
+    # @return [String] pretty analysis 
+    def pretty_analysis_for(impact)
+      final_message = ""
+
+      self.tables.keys.each do |table|
+        table_message = table_pretty_analysis(table, {impact => true})
+
+        final_message += "table: #{table}\n#{table_message}\n" if table_message.present?
+      end
+
+      final_message
+    end
+
+    # to get analysis in pretty format with warnings and suggestions for a table
+    # @param table [String] - table name
+    # @param opts [Hash] - possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
+    # @return [String] pretty analysis 
+    def table_pretty_analysis(table, opts)
+      table_message = ""
+
+      self.tables.dig(table, "analysis").each do |column, column_analysis|
+        tags_message = ""
+        column_analysis.dig("tags").each do |tag, tag_analysis|
+          next unless opts.dig(tag_analysis.dig("impact")).present?
+
+          tags_message += tag_pretty_analysis(table, column, tag)
+        end
+
+        table_message += "column: #{column}\n#{tags_message}" if tags_message.present?
+      end
+
+      table_message
+    end
+
+    private
+
+    def tag_pretty_analysis(table, column, tag)
+      tag_message = ""
+
+      opts = { "table" => table, "column" => column, "tag" => tag }
+      message = dynamic_message(opts.merge({ "type" => "message" }))
+      suggestion = dynamic_message(opts.merge({ "type" => "suggestion" }))
+      tag_message += "impact: #{impact(opts.merge({ "colours" => true }))}\n"
+      tag_message += "message: #{message}\n"
+      tag_message += "suggestion: #{suggestion}\n" if suggestion.present?
+
+      tag_message
+    end
+  end
+end

data/lib/query_police/config.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module QueryPolice
+  # This class is used for configuration of query police
+  class Config
+    def initialize(detailed, rules_path)
+      @detailed = detailed
+      @rules_path = rules_path
+    end
+
+    def detailed?
+      @detailed.present?
+    end
+
+    attr_accessor :detailed, :rules_path
+  end
+end

data/lib/query_police/constants.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module QueryPolice
+  module Constants
+    DEFAULT_COLUMN_RULES = {
+      "value_type" => "string"
+    }.freeze
+    DEFAULT_DETAILED = true
+    DEFAULT_LOGGER_CONFIG = {
+      "negative" => true
+    }.freeze
+    DEFAULT_RULES_PATH =  File.join(File.dirname(__FILE__), "rules.json")
+  end
+end

data/lib/query_police/explain.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "helper"
+
+module QueryPolice
+  # This module provides tools to explain queries and ActiveRecord::Relation
+  module Explain
+    # to get explain result in parsable format
+    # @param relation [ActiveRecord::Relation, String] active record relation or raw sql query
+    # @return [Array] parsed_result - array of hashes representing EXPLAIN result for each row
+    def full_explain(relation, detailed = true)
+      explain_result = explain(relation)
+      return explain_result unless detailed
+
+      detailed_explain_result = detailed_explain(relation)
+
+      [*explain_result.keys, *detailed_explain_result.keys].uniq.map do |key|
+        (
+          explain_result.dig(key)&.merge(
+            detailed_explain_result.dig(key) || {}
+          ) || detailed_explain_result.dig(key)
+        )
+      end
+    end
+
+    # to get explain result in parsable format using "EXPLAIN <query>"
+    # @param relation [ActiveRecord::Relation, String] active record relation or raw sql query
+    # @return [Array] parsed_result - array of hashes representing EXPLAIN result for each row
+    def explain(relation)
+      query = load_query(relation)
+      explain_result = ActiveRecord::Base.connection.execute("EXPLAIN #{query}")
+      parsed_result = {}
+
+      explain_result.each(as: :json) do |ele|
+        parsed_result[ele.dig("table")] = ele
+      end
+
+      parsed_result
+    end
+
+    # to get detailed explain result in parsable format using "EXPLAIN format=JSON <query>"
+    # @param relation [ActiveRecord::Relation, String] active record relation or raw sql query
+    # @param prefix [String] prefix to append before each key "prefix#<key>"
+    # @return [Array] parsed_result - array of flatten hashes representing EXPLAIN result for each row
+    def detailed_explain(relation, prefix = "detailed")
+      query = load_query(relation)
+      explain_result = ActiveRecord::Base.connection.execute("EXPLAIN format=json #{query}")
+      explain_result = parse_detailed_explain(explain_result)
+
+      explain_result.map { |ele| [ele.dig("table_name"), Helper.flatten_hash(ele, prefix)] }.to_h
+    end
+
+    class << self
+      private
+
+      def load_query(relation)
+        relation.class.name == "ActiveRecord::Relation" ? relation.to_sql : relation
+      end
+
+      def parse_detailed_explain(explain_result)
+        parsed_result = JSON.parse(explain_result&.first&.first || "{}").dig("query_block")
+        return parsed_result.dig("nested_loop").map { |e| e.dig("table") } if parsed_result.key?("nested_loop")
+
+        parsed_result.key?("table") ? [parsed_result.dig("table")] : []
+      end
+    end
+
+    module_function :full_explain, :explain, :detailed_explain
+  end
+end

data/lib/query_police/helper.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module QueryPolice
+  # This module define helper methods for query police
+  module Helper
+    def flatten_hash(hash, prefix_key = "")
+      flat_hash = {}
+
+      hash.each do |key, value|
+        key = prefix_key.present? ? "#{prefix_key}##{key}" : key.to_s
+
+        flat_hash.merge!(value.is_a?(Hash) ? flatten_hash(value, key) : { key => value })
+      end
+
+      flat_hash
+    end
+
+    def logger(message, type="info")
+      if defined?(Rails) && Rails.logger
+        Rails.logger.send(type, message)
+      else
+        puts "#{type.upcase}: #{message}"
+      end
+    end
+
+    def load_config(rules_path)
+      unless File.exists?(rules_path)
+        raise Error.new(
+          "Failed to load the rule file from '#{rules_path}'. " \
+          "The file may be missing or there is a problem with the path. " \
+          "Please ensure that the file exists and the path is correct."
+        )
+      end
+
+      rules_config = JSON.parse(File.read(rules_path))
+
+      rules_config
+    end
+
+    module_function :flatten_hash, :logger, :load_config
+  end
+
+  private_constant :Helper
+end

data/lib/query_police/rules.json

@@ -0,0 +1,171 @@
+{
+  "select_type": {
+    "description": "Type of select used in the table.",
+    "value_type": "string",
+    "rules": {
+      "SIMPLE": {
+        "impact": "positive",
+        "message": "A simple query without subqueries or unions.",
+        "suggestion": ""
+      }
+    }
+  },
+  "type": {
+    "description": "Join used in the query for a specific table.",
+    "value_type": "string",
+    "rules": {
+      "system": {
+        "impact": "positive",
+        "message": "Table has zero or one row, no change required.",
+        "suggestion": ""
+      },
+      "const": {
+        "impact": "positive",
+        "message": "Table has only one indexed matching row, fastest join type.",
+        "suggestion": ""
+      },
+      "eq_ref": {
+        "impact": "positive",
+        "message": "All index parts used in join and index is primary_key or unique not null.",
+        "suggestion": ""
+      },
+      "ref": {
+        "impact": "caution",
+        "message": "All matching rows of an indexed column read for each combination of rows from previous table.",
+        "suggestion": "Ensure the referenced column is indexed and look for null values, dupilcates. Upgrade to eq_ref join type if possible.\nYou can acheive eq_ref by adding unique and not null to the index - $key used in $table table table."
+      },
+      "fulltext": {
+        "impact": "caution",
+        "message": "Join uses table FULLTEXT index, index key used - $key.",
+        "suggestion": "Should only be used for text heavy columns."
+      },
+      "ref_or_null": {
+        "impact": "caution",
+        "message": "Using ref index with Null value in $table table.",
+        "suggestion": "Please check if you can upgrade to eq_ref, you can acheive eq_ref by adding unique and not null to the index - $key used in $table table."
+      },
+      "index_merge": {
+        "impact": "caution",
+        "message": "Join uses list of indexes, keys used: $key.",
+        "suggestion": "Slow if the indexes are not well-chosen or if there are too many indexes being used."
+      },
+      "range": {
+        "impact": "caution",
+        "message": "Index used to find matching rows in specific range.",
+        "suggestion": "Please check the range it should not be too broad."
+      },
+      "index": {
+        "impact": "caution",
+        "message": "Entire index tree scanned to find matching rows.",
+        "suggestion": "Can be slow for large indexes(Your key length: $key_len), use carefully."
+      },
+      "ALL": {
+        "impact": "negative",
+        "message": "Entire $table table is scanned to find matching rows, you have $amount_possible_keys possible keys to use.",
+        "suggestion": "Use index here. You can use index from possible key: $possible_keys or add new one to $table table as per the requirements."
+      }
+    }
+  },
+  "rows": {
+    "description": "Estimated number of rows scanned to find matching rows.",
+    "value_type": "number",
+    "rules": {
+      "threshold": {
+        "amount": 100,
+        "impact": "negative",
+        "message": "$value rows are being scanned per join for $table table.",
+        "suggestion": "Please see if it is possible to use index from $possible_keys or add new one to $table table as per the requirements to reduce the number of rows scanned."
+      }
+    }
+  },
+  "possible_keys": {
+    "description": "Index keys possible for a specifc table",
+    "value_type": "array",
+    "delimiter": ",",
+    "rules": {
+      "absent": {
+        "impact": "negative",
+        "message": "There are no possible keys for $table table to be used, can result into full scan",
+        "suggestion": "Please add index keys for $table table"
+      },
+      "threshold": {
+        "amount": 5,
+        "impact": "negative",
+        "message": "There are $amount possible keys for $table table, having too many index keys can be unoptimal",
+        "suggestion": "Please check if there are extra indexes in $table table."
+      }
+    }
+  },
+  "key": {
+    "description": "",
+    "value_type": "string",
+    "rules": {
+      "absent": {
+        "impact": "negative",
+        "message": "There is no index key used for $table table, and can result into full scan of the $table table",
+        "suggestion": "Please use index from possible_keys: $possible_keys or add new one to $table table as per the requirements."
+      }
+    }
+  },
+  "key_len": {
+    "description": "Length of the key index used",
+    "value_type": "number",
+    "rules": {}
+  },
+  "filtered": {
+    "description": "Indicates percentage of rows appearing from the total.",
+    "value_type": "number",
+    "rules": {}
+  },
+  "extra": {
+    "description": "Additional information about the plan",
+    "value_type": "array",
+    "delimiter": ";",
+    "rules": {
+      "Using temporary": {
+        "impact": "",
+        "message": "",
+        "suggestion": ""
+      },
+      "Using filesort": {
+        "impact": "negative",
+        "message": "A file-based algorithm in being applied over your result, This can be inefficient and result into long query time.",
+        "suggestion": "Please ensure either result set is small or use proper index."
+      },
+      "Using join buffer": {
+        "impact": "",
+        "message": "",
+        "suggestion": ""
+      },
+      "Using index condition": {
+        "impact": "",
+        "message": "",
+        "suggestion": ""
+      }
+    }
+  },
+  "detailed#used_columns": {
+    "description": "",
+    "value_type": "array",
+    "rules": {
+      "threshold": {
+        "amount": 7,
+        "impact": "negative",
+        "message": "You have selected $amount columns, You should not select too many columns.",
+        "suggestion": "Please only select required columns."
+      }
+    }
+  },
+  "cardinality": {
+    "description": "",
+    "value_type": "number",
+    "rules": {
+      "threshold": {
+        "amount": 100,
+        "impact": "negative",
+        "message": "The cardinality of table is $amount, and its too high.",
+        "suggestion": "Please use proper index, query only requried data and ensure you are using proper joins."
+      }
+    }
+  }
+}

data/lib/query_police/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module QueryPolice
+  VERSION = "0.1.0.beta"
+end