query_police 0.1.3.beta → 0.1.5.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,13 @@
1
+ {
2
+ "key": {
3
+ "description": "index key used for the table",
4
+ "value_type": "string",
5
+ "rules": {
6
+ "absent": {
7
+ "impact": "negative",
8
+ "message": "There is no index key used for $table table, and can result into full scan of the $table table",
9
+ "suggestion": "Please use index from possible_keys: $possible_keys or add new one to $table table as per the requirements."
10
+ }
11
+ }
12
+ }
13
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "type": {
3
+ "description": "Join used in the query for a specific table.",
4
+ "value_type": "string",
5
+ "rules": {
6
+ "system": {
7
+ "impact": "positive",
8
+ "message": "Table has zero or one row, no change required.",
9
+ "suggestion": ""
10
+ },
11
+ "ALL": {
12
+ "impact": "negative",
13
+ "message": "Entire $table table is scanned to find matching rows, you have $amount_possible_keys possible keys to use.",
14
+ "suggestion": "Use index here. You can use index from possible key: $possible_keys or add new one to $table table as per the requirements.",
15
+ "debt": {
16
+ "value": 200,
17
+ "type": "base"
18
+ }
19
+ }
20
+ }
21
+ }
22
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "detailed#used_columns": {
3
+ "description": "",
4
+ "value_type": "array",
5
+ "rules": {
6
+ "threshold": {
7
+ "amount": 7,
8
+ "impact": "negative",
9
+ "message": "You have selected $amount columns, You should not select too many columns.",
10
+ "suggestion": "Please only select required columns.",
11
+ "debt": {
12
+ "value": 10,
13
+ "type": "threshold_relative"
14
+ }
15
+ }
16
+ }
17
+ }
18
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "possible_keys": {
3
+ "description": "Index keys possible for a specifc table",
4
+ "value_type": "array",
5
+ "delimiter": ",",
6
+ "rules": {
7
+ "threshold": {
8
+ "amount": 5,
9
+ "impact": "negative",
10
+ "message": "There are $amount possible keys for $table table, having too many index keys can be unoptimal",
11
+ "suggestion": "Please check if there are extra indexes in $table table."
12
+ }
13
+ }
14
+ }
15
+ }
@@ -0,0 +1,11 @@
1
+ ---
2
+ key:
3
+ description: index key used for the table
4
+ value_type: string
5
+ rules:
6
+ absent:
7
+ impact: negative
8
+ message: There is no index key used for $table table, and can result into full
9
+ scan of the $table table
10
+ suggestion: 'Please use index from possible_keys: $possible_keys or add new
11
+ one to $table table as per the requirements.'
@@ -0,0 +1,18 @@
1
+ ---
2
+ type:
3
+ description: Join used in the query for a specific table.
4
+ value_type: string
5
+ rules:
6
+ system:
7
+ impact: positive
8
+ message: Table has zero or one row, no change required.
9
+ suggestion: ''
10
+ ALL:
11
+ impact: negative
12
+ message: Entire $table table is scanned to find matching rows, you have $amount_possible_keys
13
+ possible keys to use.
14
+ suggestion: 'Use index here. You can use index from possible key: $possible_keys
15
+ or add new one to $table table as per the requirements.'
16
+ debt:
17
+ value: 200
18
+ type: base
@@ -0,0 +1,13 @@
1
+ ---
2
+ detailed#used_columns:
3
+ description: ''
4
+ value_type: array
5
+ rules:
6
+ threshold:
7
+ amount: 7
8
+ impact: negative
9
+ message: You have selected $amount columns, You should not select too many columns.
10
+ suggestion: Please only select required columns.
11
+ debt:
12
+ value: 10
13
+ type: threshold_relative
@@ -0,0 +1,12 @@
1
+ ---
2
+ possible_keys:
3
+ description: Index keys possible for a specifc table
4
+ value_type: array
5
+ delimiter: ","
6
+ rules:
7
+ threshold:
8
+ amount: 5
9
+ impact: negative
10
+ message: There are $amount possible keys for $table table, having too many index
11
+ keys can be unoptimal
12
+ suggestion: Please check if there are extra indexes in $table table.
@@ -6,38 +6,38 @@ module QueryPolice
6
6
  module Analyse
7
7
  def table(table, summary, rules_config)
8
8
  table_analysis = {}
9
- table_score = 0
9
+ table_debt = 0
10
10
 
11
11
  table.each do |column, value|
12
12
  summary = add_summary(summary, column, value)
13
13
  next unless rules_config.dig(column).present?
14
14
 
15
15
  table_analysis.merge!({ column => apply_rules(rules_config.dig(column), value) })
16
- table_score += table_analysis.dig(column, "tags").map { |_, tag| tag.dig("score") }.sum.to_f
16
+ table_debt += table_analysis.dig(column, "tags").map { |_, tag| tag.dig("debt") }.sum.to_f
17
17
  end
18
18
 
19
- [table_analysis, summary, table_score]
19
+ [table_analysis, summary, table_debt]
20
20
  end
21
21
 
22
22
  def generate_summary(rules_config, summary)
23
23
  summary_analysis = {}
24
- summary_score = 0
24
+ summary_debt = 0
25
25
 
26
26
  summary.each do |column, value|
27
27
  next unless rules_config.dig(column).present?
28
28
 
29
29
  summary_analysis.merge!({ column => apply_rules(rules_config.dig(column), value) })
30
- summary_score += summary_analysis.dig(column, "tags").map { |_, tag| tag.dig("score") }.sum.to_f
30
+ summary_debt += summary_analysis.dig(column, "tags").map { |_, tag| tag.dig("debt") }.sum.to_f
31
31
  end
32
32
 
33
- [summary_analysis, summary_score]
33
+ [summary_analysis, summary_debt]
34
34
  end
35
35
 
36
36
  class << self
37
37
  private
38
38
 
39
39
  def add_summary(summary, column_name, value)
40
- summary["cardinality"] = (summary.dig("cardinality") || 1) + value.to_f if column_name.eql?("rows")
40
+ summary["cardinality"] = (summary.dig("cardinality") || 1) * value.to_f if column_name.eql?("rows")
41
41
 
42
42
  summary
43
43
  end
@@ -54,7 +54,7 @@ module QueryPolice
54
54
  next unless tag_rule.present?
55
55
 
56
56
  column_analyse["tags"].merge!(
57
- { tag => Transform.tag_rule(tag_rule).merge!({ "score" => generate_score(tag_rule, amount) }) }
57
+ { tag => Transform.tag_rule(tag_rule).merge!({ "debt" => generate_debt(tag_rule, amount) }) }
58
58
  )
59
59
  end
60
60
 
@@ -69,7 +69,7 @@ module QueryPolice
69
69
  if threshold_rule.present? && amount >= threshold_rule.dig("amount")
70
70
  return {
71
71
  "threshold" => Transform.tag_rule(threshold_rule).merge(
72
- { "amount" => amount, "score" => generate_score(threshold_rule, amount) }
72
+ { "amount" => amount, "debt" => generate_debt(threshold_rule, amount) }
73
73
  )
74
74
  }
75
75
  end
@@ -77,16 +77,16 @@ module QueryPolice
77
77
  {}
78
78
  end
79
79
 
80
- def generate_score(tag_rule, amount)
81
- score = tag_rule.dig("score", "value")
80
+ def generate_debt(tag_rule, amount)
81
+ debt = tag_rule.dig("debt", "value")
82
82
 
83
- case tag_rule.dig("score", "type").to_s
83
+ case tag_rule.dig("debt", "type").to_s
84
84
  when "base"
85
- score.to_f
85
+ debt.to_f
86
86
  when "relative"
87
- amount.to_f * score.to_f
88
- when "treshold_relative"
89
- (amount - tag_rule.dig("amount")).to_f * score.to_f
87
+ amount.to_f * debt.to_f
88
+ when "threshold_relative"
89
+ (amount - tag_rule.dig("amount")).to_f * debt.to_f
90
90
  else
91
91
  0
92
92
  end
@@ -54,13 +54,13 @@ module QueryPolice
54
54
  opts.dig("colours").present? ? impact.send(IMPACTS.dig(impact, "colour")) : impact
55
55
  end
56
56
 
57
- def score(opts)
57
+ def debt(opts)
58
58
  table, column, tag = opts.values_at("table", "column", "tag")
59
59
 
60
60
  impact = query_analytic.dig(table, "analysis", column, "tags", tag, "impact")
61
- score = query_analytic.dig(table, "analysis", column, "tags", tag, "score")
61
+ debt = query_analytic.dig(table, "analysis", column, "tags", tag, "debt")
62
62
 
63
- opts.dig("colours").present? ? score.to_s.send(IMPACTS.dig(impact, "colour")) : score
63
+ opts.dig("colours").present? ? debt.to_s.send(IMPACTS.dig(impact, "colour")) : debt
64
64
  end
65
65
 
66
66
  def table(opts)
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "analysis/dynamic_message"
4
+ require_relative "helper"
4
5
 
5
6
  module QueryPolice
6
7
  # This class is used to store analysis of a query and provide methods over them
@@ -20,7 +21,7 @@ module QueryPolice
20
21
  # "users" => {
21
22
  # "id" => 1,
22
23
  # "name" => "users",
23
- # "score" => 100.0,
24
+ # "debt" => 100.0,
24
25
  # "analysis" => {
25
26
  # "type" => {
26
27
  # "value" => all",
@@ -29,7 +30,7 @@ module QueryPolice
29
30
  # "impact" => "negative",
30
31
  # "warning" => "warning to represent the issue",
31
32
  # "suggestions" => "some follow up suggestions",
32
- # "score" => 100.0
33
+ # "debt" => 100.0
33
34
  # }
34
35
  # }
35
36
  # }
@@ -43,28 +44,28 @@ module QueryPolice
43
44
  # "amount" => 10,
44
45
  # "warning" => "warning to represent the issue",
45
46
  # "suggestions" => "some follow up suggestions",
46
- # "score" => 100.0
47
+ # "debt" => 100.0
47
48
  # }
48
49
  # }
49
- def initialize
50
+ def initialize(footer: nil)
51
+ @footer = footer || ""
52
+ @summary = {}
53
+ @summary_debt = 0
50
54
  @table_count = 0
55
+ @table_debt = 0
51
56
  @tables = {}
52
- @table_score = 0
53
- @summary = {}
54
- @summary_score = 0
55
57
  end
56
58
 
57
- attr_accessor :tables, :summary
59
+ attr_accessor :summary, :tables
58
60
 
59
61
  # register a table analysis in analysis object
60
62
  # @param name [String] name of the table
61
63
  # @param table_analysis [Hash] analysis of a table
62
- # @param score [Integer] score for that table
63
64
  # Eg.
64
65
  # {
65
66
  # "id" => 1,
66
67
  # "name" => "users",
67
- # "score" => 100.0
68
+ # "debt" => 100.0
68
69
  # "analysis" => {
69
70
  # "type" => [
70
71
  # {
@@ -72,76 +73,102 @@ module QueryPolice
72
73
  # "impact" => "negative",
73
74
  # "warning" => "warning to represent the issue",
74
75
  # "suggestions" => "some follow up suggestions",
75
- # "score" => 100.0
76
+ # "debt" => 100.0
76
77
  # }
77
78
  # ]
78
79
  # }
79
80
  # }
80
- def register_table(name, table_analysis, score)
81
+ # @param debt [Integer] debt for that table
82
+ def register_table(name, table_analysis, debt)
81
83
  @table_count += 1
82
84
  tables.merge!(
83
85
  {
84
86
  name => {
85
87
  "id" => @table_count,
86
88
  "name" => name,
87
- "score" => score,
89
+ "debt" => debt,
88
90
  "analysis" => table_analysis
89
91
  }
90
92
  }
91
93
  )
92
94
 
93
- @table_score += score
95
+ @table_debt += debt
94
96
  end
95
97
 
96
98
  # register summary based in different attributes
97
99
  # @param summary [Hash] hash of summary of analysis
98
- def register_summary(summary, score)
100
+ def register_summary(summary, debt)
99
101
  self.summary.merge!(summary)
100
- @summary_score += score
102
+ @summary_debt += debt
101
103
  end
102
104
 
103
105
  # to get analysis in pretty format with warnings and suggestions
104
- # @param opts [Hash] - possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
106
+ # @param opts [Hash] - options
107
+ # possible keys
108
+ # [
109
+ # positive: <boolean>,
110
+ # negative: <boolean>,
111
+ # caution: <boolean>,
112
+ # wrap_width: <integer>,
113
+ # skip_footer: <boolean>
114
+ # ]
105
115
  # @return [String] pretty analysis
106
- def pretty_analysis(opts)
116
+ def pretty_analysis(opts = { "negative" => true, "caution" => true })
107
117
  final_message = ""
108
118
  opts = opts.with_indifferent_access
109
119
 
110
120
  opts.slice(*IMPACTS.keys).each do |impact, value|
111
- final_message += pretty_analysis_for(impact) if value.present?
121
+ opts_ = opts.slice("wrap_width").merge({ "skip_footer" => true })
122
+ final_message += pretty_analysis_for(impact, opts_) if value.present?
112
123
  end
113
124
 
114
- final_message
125
+ opts.dig("skip_footer").present? ? final_message : final_message + @footer
115
126
  end
116
127
 
117
128
  # to get analysis in pretty format with warnings and suggestions for a impact
118
129
  # @param impact [String]
130
+ # @param opts [Hash] - options
131
+ # possible keys
132
+ # [
133
+ # wrap_width: <integer>
134
+ # skip_footer: <boolean>
135
+ # ]
119
136
  # @return [String] pretty analysis
120
- def pretty_analysis_for(impact)
121
- final_message = "query_score: #{query_score}\n\n"
137
+ def pretty_analysis_for(impact, opts = {})
138
+ final_message = ""
122
139
 
123
140
  query_analytic.each_key do |table|
124
- table_message = query_pretty_analysis(table, { impact => true })
141
+ table_message = query_pretty_analysis(table, { impact => true }.merge(opts))
125
142
 
126
143
  final_message += "#{table_message}\n" if table_message.present?
127
144
  end
128
145
 
129
- final_message
146
+ return final_message unless final_message.present?
147
+
148
+ final_message = "query_debt: #{query_debt}\n\n#{final_message}"
149
+ opts.dig("skip_footer").present? ? final_message : final_message + @footer
130
150
  end
131
151
 
132
- # to get the final score
133
- def query_score
134
- @table_score + @summary_score
152
+ # to get the final debt
153
+ def query_debt
154
+ @table_debt + @summary_debt
135
155
  end
136
156
 
137
157
  # to get analysis in pretty format with warnings and suggestions for a table
138
158
  # @param table [String] - table name
139
- # @param opts [Hash] - possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
159
+ # @param opts [Hash] - options
160
+ # possible keys
161
+ # [
162
+ # positive: <boolean>,
163
+ # negative: <boolean>,
164
+ # caution: <boolean>,
165
+ # wrap_width: <integer>
166
+ # ]
140
167
  # @return [String] pretty analysis
141
168
  def query_pretty_analysis(table, opts)
142
169
  table_analytics = Terminal::Table.new(title: table)
143
170
  table_analytics_present = false
144
- table_analytics.add_row(["score", query_analytic.dig(table, "score")])
171
+ table_analytics.add_row(["debt", query_analytic.dig(table, "debt")])
145
172
 
146
173
  opts = opts.with_indifferent_access
147
174
 
@@ -166,7 +193,7 @@ module QueryPolice
166
193
  query_analytic.dig(table, "analysis", column, "tags").each do |tag, tag_analysis|
167
194
  next unless opts.dig(tag_analysis.dig("impact")).present?
168
195
 
169
- column_analytics += tag_analytic(table, column, tag)
196
+ column_analytics += tag_analytic(table, column, tag, opts)
170
197
  end
171
198
 
172
199
  column_analytics
@@ -176,22 +203,24 @@ module QueryPolice
176
203
  tables.merge(
177
204
  "summary" => {
178
205
  "name" => "summary",
179
- "score" => @summary_score,
206
+ "debt" => @summary_debt,
180
207
  "analysis" => summary
181
208
  }
182
209
  )
183
210
  end
184
211
 
185
- def tag_analytic(table, column, tag)
212
+ def tag_analytic(table, column, tag, opts)
186
213
  tag_message = []
187
214
 
188
- opts = { "table" => table, "column" => column, "tag" => tag }
189
- message = dynamic_message(opts.merge({ "type" => "message" }))
190
- suggestion = dynamic_message(opts.merge({ "type" => "suggestion" }))
191
- tag_message << ["impact", impact(opts.merge({ "colours" => true }))]
192
- tag_message << ["tag_score", score(opts.merge({ "colours" => true }))]
193
- tag_message << ["message", message]
194
- tag_message << ["suggestion", suggestion] if suggestion.present?
215
+ variable_opts = { "table" => table, "column" => column, "tag" => tag }
216
+ message = dynamic_message(variable_opts.merge({ "type" => "message" }))
217
+ suggestion = dynamic_message(variable_opts.merge({ "type" => "suggestion" }))
218
+ wrap_width = opts.dig("wrap_width")
219
+
220
+ tag_message << ["impact", impact(variable_opts.merge({ "colours" => true }))]
221
+ tag_message << ["tag_debt", debt(variable_opts.merge({ "colours" => true }))]
222
+ tag_message << ["message", Helper.word_wrap(message, wrap_width)]
223
+ tag_message << ["suggestion", Helper.word_wrap(suggestion, wrap_width)] if suggestion.present?
195
224
 
196
225
  tag_message
197
226
  end
@@ -1,17 +1,22 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "constants"
4
+
3
5
  module QueryPolice
4
6
  # This class is used for configuration of query police
5
7
  class Config
6
- def initialize(detailed, rules_path)
7
- @detailed = detailed
8
- @rules_path = rules_path
8
+ def initialize
9
+ @action_enabled = Constants::DEFAULT_ACTION_ENABLED
10
+ @analysis_footer = Constants::DEFAULT_ANALYSIS_FOOTER
11
+ @logger_options = Constants::DEFAULT_LOGGER_OPTIONS
12
+ @rules_path = Constants::DEFAULT_RULES_PATH
13
+ @verbosity = Constants::DEFAULT_VERBOSITY
9
14
  end
10
15
 
11
- def detailed?
12
- @detailed.present?
16
+ def action_enabled?
17
+ @action_enabled.present?
13
18
  end
14
19
 
15
- attr_accessor :detailed, :rules_path
20
+ attr_accessor :action_enabled, :analysis_footer, :logger_options, :rules_path, :verbosity
16
21
  end
17
22
  end
@@ -2,13 +2,16 @@
2
2
 
3
3
  module QueryPolice
4
4
  module Constants
5
+ DEFAULT_ANALYSIS_FOOTER = ""
5
6
  DEFAULT_COLUMN_RULES = {
6
7
  "value_type" => "string"
7
8
  }.freeze
8
- DEFAULT_DETAILED = true
9
- DEFAULT_LOGGER_CONFIG = {
10
- "negative" => true
9
+ DEFAULT_ACTION_ENABLED = true
10
+ DEFAULT_LOGGER_OPTIONS = {
11
+ "negative" => true,
12
+ "caution" => true
11
13
  }.freeze
12
14
  DEFAULT_RULES_PATH = File.join(File.dirname(__FILE__), "rules.json")
15
+ DEFAULT_VERBOSITY = "detailed"
13
16
  end
14
17
  end
@@ -5,12 +5,15 @@ require_relative "helper"
5
5
  module QueryPolice
6
6
  # This module provides tools to explain queries and ActiveRecord::Relation
7
7
  module Explain
8
+ DETAILED_VERBOSITY = "detailed"
9
+
8
10
  # to get explain result in parsable format
9
11
  # @param relation [ActiveRecord::Relation, String] active record relation or raw sql query
12
+ # @param verbosity [Symbol] mode to define which EXPLAIN result should be inlcuded in final result
10
13
  # @return [Array] parsed_result - array of hashes representing EXPLAIN result for each row
11
- def full_explain(relation, detailed = true)
14
+ def full_explain(relation, verbosity = nil)
12
15
  explain_result = explain(relation)
13
- return explain_result unless detailed
16
+ return explain_result.values unless verbosity.to_s.eql?(DETAILED_VERBOSITY)
14
17
 
15
18
  detailed_explain_result = detailed_explain(relation)
16
19
 
@@ -59,10 +62,20 @@ module QueryPolice
59
62
 
60
63
  def parse_detailed_explain(explain_result)
61
64
  parsed_result = JSON.parse(explain_result&.first&.first || "{}").dig("query_block")
65
+ parsed_result = parse_detailed_explain_operations(parsed_result)
66
+
62
67
  return parsed_result.dig("nested_loop").map { |e| e.dig("table") } if parsed_result.key?("nested_loop")
63
68
 
64
69
  parsed_result.key?("table") ? [parsed_result.dig("table")] : []
65
70
  end
71
+
72
+ def parse_detailed_explain_operations(parsed_result)
73
+ parsed_result = parsed_result.dig("ordering_operation") || parsed_result
74
+ parsed_result = parsed_result.dig("grouping_operation") || parsed_result
75
+ parsed_result = parsed_result.dig("duplicates_removal") || parsed_result
76
+
77
+ parsed_result
78
+ end
66
79
  end
67
80
 
68
81
  module_function :full_explain, :explain, :detailed_explain
@@ -4,6 +4,8 @@
4
4
  module QueryPolice
5
5
  # This module define helper methods for query police
6
6
  module Helper
7
+ DEFAULT_WORD_WRAP_WIDTH = 100
8
+
7
9
  def flatten_hash(hash, prefix_key = "")
8
10
  flat_hash = {}
9
11
 
@@ -31,10 +33,31 @@ module QueryPolice
31
33
  "Please ensure that the file exists and the path is correct."
32
34
  end
33
35
 
34
- JSON.parse(File.read(rules_path))
36
+ case File.extname(rules_path)
37
+ when ".yaml", ".yml"
38
+ YAML.safe_load(File.read(rules_path))
39
+ when ".json"
40
+ JSON.parse(File.read(rules_path))
41
+ else
42
+ raise Error, "'#{File.extname(rules_path)}' extension is not supported for rules."
43
+ end
44
+ end
45
+
46
+ def word_wrap(string, width = DEFAULT_WORD_WRAP_WIDTH)
47
+ width ||= DEFAULT_WORD_WRAP_WIDTH
48
+ words = string.split
49
+ wrapped_string = ""
50
+
51
+ words.each do |word|
52
+ last_line_size = (wrapped_string.split("\n")[-1]&.size || 0)
53
+ wrapped_string = wrapped_string.strip + "\n" if (last_line_size + word.size) > width
54
+ wrapped_string += "#{word} "
55
+ end
56
+
57
+ wrapped_string.strip
35
58
  end
36
59
 
37
- module_function :flatten_hash, :logger, :load_config
60
+ module_function :flatten_hash, :logger, :load_config, :word_wrap
38
61
  end
39
62
 
40
63
  private_constant :Helper