query_police 0.1.4.beta → 0.1.6.beta

data/examples/rules/json/absent_rule.json

@@ -0,0 +1,13 @@
+{
+  "key": {
+    "description": "index key used for the table",
+    "value_type": "string",
+    "rules": {
+      "absent": {
+        "impact": "negative",
+        "message": "There is no index key used for $table table, and can result into full scan of the $table table",
+        "suggestion": "Please use index from possible_keys: $possible_keys or add new one to $table table as per the requirements."
+      }
+    }
+  }
+}

data/examples/rules/json/basic_rule.json

@@ -0,0 +1,22 @@
+{
+  "type": {
+    "description": "Join used in the query for a specific table.",
+    "value_type": "string",
+    "rules": {
+      "system": {
+        "impact": "positive",
+        "message": "Table has zero or one row, no change required.",
+        "suggestion": ""
+      },
+      "ALL": {
+        "impact": "negative",
+        "message": "Entire $table table is scanned to find matching rows, you have $amount_possible_keys possible keys to use.",
+        "suggestion": "Use index here. You can use index from possible key: $possible_keys or add new one to $table table as per the requirements.",
+        "debt": {
+          "value": 200,
+          "type": "base"
+        }
+      }
+    }
+  }
+}

data/examples/rules/json/complex_detailed_rule.json

@@ -0,0 +1,18 @@
+{
+  "detailed#used_columns": {
+    "description": "",
+    "value_type": "array",
+    "rules": {
+      "threshold": {
+        "amount": 7,
+        "impact": "negative",
+        "message": "You have selected $amount columns, You should not select too many columns.",
+        "suggestion": "Please only select required columns.",
+        "debt": {
+          "value": 10,
+          "type": "threshold_relative"
+        }
+      }
+    }
+  }
+}

data/examples/rules/json/threshold_rule.json

@@ -0,0 +1,15 @@
+{
+  "possible_keys": {
+    "description": "Index keys possible for a specifc table",
+    "value_type": "array",
+    "delimiter": ",",
+    "rules": {
+      "threshold": {
+        "amount": 5,
+        "impact": "negative",
+        "message": "There are $amount possible keys for $table table, having too many index keys can be unoptimal",
+        "suggestion": "Please check if there are extra indexes in $table table."
+      }
+    }
+  }
+}

data/examples/rules/yaml/absent_rule.yml

@@ -0,0 +1,11 @@
+---
+key:
+  description: index key used for the table
+  value_type: string
+  rules:
+    absent:
+      impact: negative
+      message: There is no index key used for $table table, and can result into full
+        scan of the $table table
+      suggestion: 'Please use index from possible_keys: $possible_keys or add new
+        one to $table table as per the requirements.'

data/examples/rules/yaml/basic_rule.yml

@@ -0,0 +1,18 @@
+---
+type:
+  description: Join used in the query for a specific table.
+  value_type: string
+  rules:
+    system:
+      impact: positive
+      message: Table has zero or one row, no change required.
+      suggestion: ''
+    ALL:
+      impact: negative
+      message: Entire $table table is scanned to find matching rows, you have $amount_possible_keys
+        possible keys to use.
+      suggestion: 'Use index here. You can use index from possible key: $possible_keys
+        or add new one to $table table as per the requirements.'
+      debt:
+        value: 200
+        type: base

data/examples/rules/yaml/complex_detailed_rule.yml

@@ -0,0 +1,13 @@
+---
+detailed#used_columns:
+  description: ''
+  value_type: array
+  rules:
+    threshold:
+      amount: 7
+      impact: negative
+      message: You have selected $amount columns, You should not select too many columns.
+      suggestion: Please only select required columns.
+      debt:
+        value: 10
+        type: threshold_relative

data/examples/rules/yaml/threshold_rule.yml

@@ -0,0 +1,12 @@
+---
+possible_keys:
+  description: Index keys possible for a specifc table
+  value_type: array
+  delimiter: ","
+  rules:
+    threshold:
+      amount: 5
+      impact: negative
+      message: There are $amount possible keys for $table table, having too many index
+        keys can be unoptimal
+      suggestion: Please check if there are extra indexes in $table table.

data/lib/query_police/analyse.rb

@@ -6,38 +6,38 @@ module QueryPolice
   module Analyse
     def table(table, summary, rules_config)
       table_analysis = {}
-      table_score = 0
+      table_debt = 0
 
       table.each do |column, value|
         summary = add_summary(summary, column, value)
         next unless rules_config.dig(column).present?
 
         table_analysis.merge!({ column => apply_rules(rules_config.dig(column), value) })
-        table_score += table_analysis.dig(column, "tags").map { |_, tag| tag.dig("score") }.sum.to_f
+        table_debt += table_analysis.dig(column, "tags").map { |_, tag| tag.dig("debt") }.sum.to_f
       end
 
-      [table_analysis, summary, table_score]
+      [table_analysis, summary, table_debt]
     end
 
     def generate_summary(rules_config, summary)
       summary_analysis = {}
-      summary_score = 0
+      summary_debt = 0
 
       summary.each do |column, value|
         next unless rules_config.dig(column).present?
 
         summary_analysis.merge!({ column => apply_rules(rules_config.dig(column), value) })
-        summary_score += summary_analysis.dig(column, "tags").map { |_, tag| tag.dig("score") }.sum.to_f
+        summary_debt += summary_analysis.dig(column, "tags").map { |_, tag| tag.dig("debt") }.sum.to_f
       end
 
-      [summary_analysis, summary_score]
+      [summary_analysis, summary_debt]
     end
 
     class << self
       private
 
       def add_summary(summary, column_name, value)
-        summary["cardinality"] = (summary.dig("cardinality") || 1) + value.to_f if column_name.eql?("rows")
+        summary["cardinality"] = (summary.dig("cardinality") || 1) * value.to_f if column_name.eql?("rows")
 
         summary
       end
@@ -54,7 +54,7 @@ module QueryPolice
           next unless tag_rule.present?
 
           column_analyse["tags"].merge!(
-            { tag => Transform.tag_rule(tag_rule).merge!({ "score" => generate_score(tag_rule, amount) }) }
+            { tag => Transform.tag_rule(tag_rule).merge!({ "debt" => generate_debt(tag_rule, amount) }) }
           )
         end
 
@@ -69,7 +69,7 @@ module QueryPolice
         if threshold_rule.present? && amount >= threshold_rule.dig("amount")
           return {
             "threshold" => Transform.tag_rule(threshold_rule).merge(
-              { "amount" => amount, "score" => generate_score(threshold_rule, amount) }
+              { "amount" => amount, "debt" => generate_debt(threshold_rule, amount) }
             )
           }
         end
@@ -77,16 +77,16 @@ module QueryPolice
         {}
       end
 
-      def generate_score(tag_rule, amount)
-        score = tag_rule.dig("score", "value")
+      def generate_debt(tag_rule, amount)
+        debt = tag_rule.dig("debt", "value")
 
-        case tag_rule.dig("score", "type").to_s
+        case tag_rule.dig("debt", "type").to_s
         when "base"
-          score.to_f
+          debt.to_f
         when "relative"
-          amount.to_f * score.to_f
-        when "treshold_relative"
-          (amount - tag_rule.dig("amount")).to_f * score.to_f
+          amount.to_f * debt.to_f
+        when "threshold_relative"
+          (amount - tag_rule.dig("amount")).to_f * debt.to_f
         else
           0
         end

data/lib/query_police/analysis/dynamic_message.rb

@@ -54,13 +54,13 @@ module QueryPolice
         opts.dig("colours").present? ? impact.send(IMPACTS.dig(impact, "colour")) : impact
       end
 
-      def score(opts)
+      def debt(opts)
         table, column, tag = opts.values_at("table", "column", "tag")
 
         impact = query_analytic.dig(table, "analysis", column, "tags", tag, "impact")
-        score = query_analytic.dig(table, "analysis", column, "tags", tag, "score")
+        debt = query_analytic.dig(table, "analysis", column, "tags", tag, "debt")
 
-        opts.dig("colours").present? ? score.to_s.send(IMPACTS.dig(impact, "colour")) : score
+        opts.dig("colours").present? ? debt.to_s.send(IMPACTS.dig(impact, "colour")) : debt
       end
 
       def table(opts)

data/lib/query_police/analysis.rb

@@ -21,7 +21,7 @@ module QueryPolice
     #   "users" => {
     #     "id" => 1,
     #     "name" => "users",
-    #     "score" => 100.0,
+    #     "debt" => 100.0,
     #     "analysis" => {
     #       "type" => {
     #         "value" => all",
@@ -30,7 +30,7 @@ module QueryPolice
     #             "impact" => "negative",
     #             "warning" => "warning to represent the issue",
     #             "suggestions" => "some follow up suggestions",
-    #             "score" => 100.0
+    #             "debt" => 100.0
     #           }
     #         }
     #       }
@@ -44,28 +44,28 @@ module QueryPolice
     #      "amount" => 10,
     #      "warning" => "warning to represent the issue",
     #      "suggestions" => "some follow up suggestions",
-    #      "score" => 100.0
+    #      "debt" => 100.0
     #    }
     #  }
-    def initialize
+    def initialize(footer: nil)
+      @footer = footer || ""
+      @summary = {}
+      @summary_debt = 0
       @table_count = 0
+      @table_debt = 0
       @tables = {}
-      @table_score = 0
-      @summary = {}
-      @summary_score = 0
     end
 
-    attr_accessor :tables, :summary
+    attr_accessor :summary, :tables
 
     # register a table analysis in analysis object
     # @param name [String] name of the table
     # @param table_analysis [Hash] analysis of a table
-    # @param score [Integer] score for that table
     # Eg.
     #  {
     #    "id" => 1,
     #    "name" => "users",
-    #    "score" => 100.0
+    #    "debt" => 100.0
     #    "analysis" => {
     #      "type" => [
     #        {
@@ -73,76 +73,104 @@ module QueryPolice
     #          "impact" => "negative",
     #          "warning" => "warning to represent the issue",
     #          "suggestions" => "some follow up suggestions",
-    #          "score" => 100.0
+    #          "debt" => 100.0
     #        }
     #      ]
     #    }
     #  }
-    def register_table(name, table_analysis, score)
+    # @param debt [Integer] debt for that table
+    def register_table(name, table_analysis, debt)
       @table_count += 1
       tables.merge!(
         {
           name => {
             "id" => @table_count,
             "name" => name,
-            "score" => score,
+            "debt" => debt,
             "analysis" => table_analysis
           }
         }
       )
 
-      @table_score += score
+      @table_debt += debt
     end
 
     # register summary based in different attributes
     # @param summary [Hash] hash of summary of analysis
-    def register_summary(summary, score)
+    def register_summary(summary, debt)
       self.summary.merge!(summary)
-      @summary_score += score
+      @summary_debt += debt
     end
 
     # to get analysis in pretty format with warnings and suggestions
-    # @param opts [Hash] - possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
+    # @param opts [Hash] - options
+    # possible keys
+    # [
+    #   positive: <boolean>,
+    #   negative: <boolean>,
+    #   caution: <boolean>,
+    #   wrap_width: <integer>,
+    #   skip_footer: <boolean>
+    # ]
     # @return [String] pretty analysis
-    def pretty_analysis(opts)
+    def pretty_analysis(opts = { "negative" => true, "caution" => true })
       final_message = ""
       opts = opts.with_indifferent_access
 
       opts.slice(*IMPACTS.keys).each do |impact, value|
-        final_message += pretty_analysis_for(impact) if value.present?
+        opts_ = opts.slice("wrap_width").merge({ "skip_footer" => true })
+        final_message += pretty_analysis_for(impact, opts_) if value.present?
       end
 
-      final_message
+      return final_message unless final_message.present?
+
+      opts.dig("skip_footer").present? ? final_message : final_message + @footer
     end
 
     # to get analysis in pretty format with warnings and suggestions for a impact
     # @param impact [String]
+    # @param opts [Hash] - options
+    # possible keys
+    # [
+    #   wrap_width: <integer>
+    #   skip_footer: <boolean>
+    # ]
     # @return [String] pretty analysis
-    def pretty_analysis_for(impact)
-      final_message = "query_score: #{query_score}\n\n"
+    def pretty_analysis_for(impact, opts = {})
+      final_message = ""
 
       query_analytic.each_key do |table|
-        table_message = query_pretty_analysis(table, { impact => true })
+        table_message = query_pretty_analysis(table, { impact => true }.merge(opts))
 
         final_message += "#{table_message}\n" if table_message.present?
       end
 
-      final_message
+      return final_message unless final_message.present?
+
+      final_message = "query_debt: #{query_debt}\n\n#{final_message}"
+      opts.dig("skip_footer").present? ? final_message : final_message + @footer
     end
 
-    # to get the final score
-    def query_score
-      @table_score + @summary_score
+    # to get the final debt
+    def query_debt
+      @table_debt + @summary_debt
     end
 
     # to get analysis in pretty format with warnings and suggestions for a table
     # @param table [String] - table name
-    # @param opts [Hash] - possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
+    # @param opts [Hash] - options
+    # possible keys
+    # [
+    #   positive: <boolean>,
+    #   negative: <boolean>,
+    #   caution: <boolean>,
+    #   wrap_width: <integer>
+    # ]
     # @return [String] pretty analysis
     def query_pretty_analysis(table, opts)
       table_analytics = Terminal::Table.new(title: table)
       table_analytics_present = false
-      table_analytics.add_row(["score", query_analytic.dig(table, "score")])
+      table_analytics.add_row(["debt", query_analytic.dig(table, "debt")])
 
       opts = opts.with_indifferent_access
 
@@ -167,7 +195,7 @@ module QueryPolice
       query_analytic.dig(table, "analysis", column, "tags").each do |tag, tag_analysis|
         next unless opts.dig(tag_analysis.dig("impact")).present?
 
-        column_analytics += tag_analytic(table, column, tag)
+        column_analytics += tag_analytic(table, column, tag, opts)
       end
 
       column_analytics
@@ -177,22 +205,24 @@ module QueryPolice
       tables.merge(
         "summary" => {
           "name" => "summary",
-          "score" => @summary_score,
+          "debt" => @summary_debt,
           "analysis" => summary
         }
       )
     end
 
-    def tag_analytic(table, column, tag)
+    def tag_analytic(table, column, tag, opts)
       tag_message = []
 
-      opts = { "table" => table, "column" => column, "tag" => tag }
-      message = dynamic_message(opts.merge({ "type" => "message" }))
-      suggestion = dynamic_message(opts.merge({ "type" => "suggestion" }))
-      tag_message << ["impact", impact(opts.merge({ "colours" => true }))]
-      tag_message << ["tag_score", score(opts.merge({ "colours" => true }))]
-      tag_message << ["message", Helper.word_wrap(message)]
-      tag_message << ["suggestion", Helper.word_wrap(suggestion)] if suggestion.present?
+      variable_opts = { "table" => table, "column" => column, "tag" => tag }
+      message = dynamic_message(variable_opts.merge({ "type" => "message" }))
+      suggestion = dynamic_message(variable_opts.merge({ "type" => "suggestion" }))
+      wrap_width = opts.dig("wrap_width")
+
+      tag_message << ["impact", impact(variable_opts.merge({ "colours" => true }))]
+      tag_message << ["tag_debt", debt(variable_opts.merge({ "colours" => true }))]
+      tag_message << ["message", Helper.word_wrap(message, wrap_width)]
+      tag_message << ["suggestion", Helper.word_wrap(suggestion, wrap_width)] if suggestion.present?
 
       tag_message
     end

data/lib/query_police/config.rb

@@ -1,17 +1,22 @@
 # frozen_string_literal: true
 
+require_relative "constants"
+
 module QueryPolice
   # This class is used for configuration of query police
   class Config
-    def initialize(detailed, rules_path)
-      @detailed = detailed
-      @rules_path = rules_path
+    def initialize
+      @action_enabled = Constants::DEFAULT_ACTION_ENABLED
+      @analysis_footer = Constants::DEFAULT_ANALYSIS_FOOTER
+      @logger_options = Constants::DEFAULT_LOGGER_OPTIONS
+      @rules_path = Constants::DEFAULT_RULES_PATH
+      @verbosity = Constants::DEFAULT_VERBOSITY
     end
 
-    def detailed?
-      @detailed.present?
+    def action_enabled?
+      @action_enabled.present?
     end
 
-    attr_accessor :detailed, :rules_path
+    attr_accessor :action_enabled, :analysis_footer, :logger_options, :rules_path, :verbosity
   end
 end

data/lib/query_police/constants.rb

@@ -2,13 +2,16 @@
 
 module QueryPolice
   module Constants
+    DEFAULT_ANALYSIS_FOOTER = ""
     DEFAULT_COLUMN_RULES = {
       "value_type" => "string"
     }.freeze
-    DEFAULT_DETAILED = true
-    DEFAULT_LOGGER_CONFIG = {
-      "negative" => true
+    DEFAULT_ACTION_ENABLED = true
+    DEFAULT_LOGGER_OPTIONS = {
+      "negative" => true,
+      "caution" => true
     }.freeze
     DEFAULT_RULES_PATH = File.join(File.dirname(__FILE__), "rules.json")
+    DEFAULT_VERBOSITY = "detailed"
   end
 end

data/lib/query_police/explain.rb

@@ -5,12 +5,15 @@ require_relative "helper"
 module QueryPolice
   # This module provides tools to explain queries and ActiveRecord::Relation
   module Explain
+    DETAILED_VERBOSITY = "detailed"
+
     # to get explain result in parsable format
     # @param relation [ActiveRecord::Relation, String] active record relation or raw sql query
+    # @param verbosity [Symbol] mode to define which EXPLAIN result should be inlcuded in final result
     # @return [Array] parsed_result - array of hashes representing EXPLAIN result for each row
-    def full_explain(relation, detailed = true)
+    def full_explain(relation, verbosity = nil)
       explain_result = explain(relation)
-      return explain_result unless detailed
+      return explain_result.values unless verbosity.to_s.eql?(DETAILED_VERBOSITY)
 
       detailed_explain_result = detailed_explain(relation)
 
@@ -59,10 +62,20 @@ module QueryPolice
 
       def parse_detailed_explain(explain_result)
         parsed_result = JSON.parse(explain_result&.first&.first || "{}").dig("query_block")
+        parsed_result = parse_detailed_explain_operations(parsed_result)
+
         return parsed_result.dig("nested_loop").map { |e| e.dig("table") } if parsed_result.key?("nested_loop")
 
         parsed_result.key?("table") ? [parsed_result.dig("table")] : []
       end
+
+      def parse_detailed_explain_operations(parsed_result)
+        parsed_result = parsed_result.dig("ordering_operation") || parsed_result
+        parsed_result = parsed_result.dig("grouping_operation") || parsed_result
+        parsed_result = parsed_result.dig("duplicates_removal") || parsed_result
+
+        parsed_result
+      end
     end
 
     module_function :full_explain, :explain, :detailed_explain

data/lib/query_police/helper.rb

@@ -4,6 +4,8 @@
 module QueryPolice
   # This module define helper methods for query police
   module Helper
+    DEFAULT_WORD_WRAP_WIDTH = 100
+
     def flatten_hash(hash, prefix_key = "")
       flat_hash = {}
 
@@ -31,12 +33,20 @@ module QueryPolice
           "Please ensure that the file exists and the path is correct."
       end
 
-      JSON.parse(File.read(rules_path))
+      case File.extname(rules_path)
+      when ".yaml", ".yml"
+        YAML.safe_load(File.read(rules_path))
+      when ".json"
+        JSON.parse(File.read(rules_path))
+      else
+        raise Error, "'#{File.extname(rules_path)}' extension is not supported for rules."
+      end
     end
 
-    def word_wrap(string, width = 100)
+    def word_wrap(string, width = DEFAULT_WORD_WRAP_WIDTH)
+      width ||= DEFAULT_WORD_WRAP_WIDTH
       words = string.split
-      wrapped_string = " "
+      wrapped_string = ""
 
       words.each do |word|
         last_line_size = (wrapped_string.split("\n")[-1]&.size || 0)