query_police 0.1.3.beta → 0.1.5.beta

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,13 @@
1
+ {
2
+ "key": {
3
+ "description": "index key used for the table",
4
+ "value_type": "string",
5
+ "rules": {
6
+ "absent": {
7
+ "impact": "negative",
8
+ "message": "There is no index key used for $table table, and can result into full scan of the $table table",
9
+ "suggestion": "Please use index from possible_keys: $possible_keys or add new one to $table table as per the requirements."
10
+ }
11
+ }
12
+ }
13
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "type": {
3
+ "description": "Join used in the query for a specific table.",
4
+ "value_type": "string",
5
+ "rules": {
6
+ "system": {
7
+ "impact": "positive",
8
+ "message": "Table has zero or one row, no change required.",
9
+ "suggestion": ""
10
+ },
11
+ "ALL": {
12
+ "impact": "negative",
13
+ "message": "Entire $table table is scanned to find matching rows, you have $amount_possible_keys possible keys to use.",
14
+ "suggestion": "Use index here. You can use index from possible key: $possible_keys or add new one to $table table as per the requirements.",
15
+ "debt": {
16
+ "value": 200,
17
+ "type": "base"
18
+ }
19
+ }
20
+ }
21
+ }
22
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "detailed#used_columns": {
3
+ "description": "",
4
+ "value_type": "array",
5
+ "rules": {
6
+ "threshold": {
7
+ "amount": 7,
8
+ "impact": "negative",
9
+ "message": "You have selected $amount columns, You should not select too many columns.",
10
+ "suggestion": "Please only select required columns.",
11
+ "debt": {
12
+ "value": 10,
13
+ "type": "threshold_relative"
14
+ }
15
+ }
16
+ }
17
+ }
18
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "possible_keys": {
3
+ "description": "Index keys possible for a specifc table",
4
+ "value_type": "array",
5
+ "delimiter": ",",
6
+ "rules": {
7
+ "threshold": {
8
+ "amount": 5,
9
+ "impact": "negative",
10
+ "message": "There are $amount possible keys for $table table, having too many index keys can be unoptimal",
11
+ "suggestion": "Please check if there are extra indexes in $table table."
12
+ }
13
+ }
14
+ }
15
+ }
@@ -0,0 +1,11 @@
1
+ ---
2
+ key:
3
+ description: index key used for the table
4
+ value_type: string
5
+ rules:
6
+ absent:
7
+ impact: negative
8
+ message: There is no index key used for $table table, and can result into full
9
+ scan of the $table table
10
+ suggestion: 'Please use index from possible_keys: $possible_keys or add new
11
+ one to $table table as per the requirements.'
@@ -0,0 +1,18 @@
1
+ ---
2
+ type:
3
+ description: Join used in the query for a specific table.
4
+ value_type: string
5
+ rules:
6
+ system:
7
+ impact: positive
8
+ message: Table has zero or one row, no change required.
9
+ suggestion: ''
10
+ ALL:
11
+ impact: negative
12
+ message: Entire $table table is scanned to find matching rows, you have $amount_possible_keys
13
+ possible keys to use.
14
+ suggestion: 'Use index here. You can use index from possible key: $possible_keys
15
+ or add new one to $table table as per the requirements.'
16
+ debt:
17
+ value: 200
18
+ type: base
@@ -0,0 +1,13 @@
1
+ ---
2
+ detailed#used_columns:
3
+ description: ''
4
+ value_type: array
5
+ rules:
6
+ threshold:
7
+ amount: 7
8
+ impact: negative
9
+ message: You have selected $amount columns, You should not select too many columns.
10
+ suggestion: Please only select required columns.
11
+ debt:
12
+ value: 10
13
+ type: threshold_relative
@@ -0,0 +1,12 @@
1
+ ---
2
+ possible_keys:
3
+ description: Index keys possible for a specifc table
4
+ value_type: array
5
+ delimiter: ","
6
+ rules:
7
+ threshold:
8
+ amount: 5
9
+ impact: negative
10
+ message: There are $amount possible keys for $table table, having too many index
11
+ keys can be unoptimal
12
+ suggestion: Please check if there are extra indexes in $table table.
@@ -6,38 +6,38 @@ module QueryPolice
6
6
  module Analyse
7
7
  def table(table, summary, rules_config)
8
8
  table_analysis = {}
9
- table_score = 0
9
+ table_debt = 0
10
10
 
11
11
  table.each do |column, value|
12
12
  summary = add_summary(summary, column, value)
13
13
  next unless rules_config.dig(column).present?
14
14
 
15
15
  table_analysis.merge!({ column => apply_rules(rules_config.dig(column), value) })
16
- table_score += table_analysis.dig(column, "tags").map { |_, tag| tag.dig("score") }.sum.to_f
16
+ table_debt += table_analysis.dig(column, "tags").map { |_, tag| tag.dig("debt") }.sum.to_f
17
17
  end
18
18
 
19
- [table_analysis, summary, table_score]
19
+ [table_analysis, summary, table_debt]
20
20
  end
21
21
 
22
22
  def generate_summary(rules_config, summary)
23
23
  summary_analysis = {}
24
- summary_score = 0
24
+ summary_debt = 0
25
25
 
26
26
  summary.each do |column, value|
27
27
  next unless rules_config.dig(column).present?
28
28
 
29
29
  summary_analysis.merge!({ column => apply_rules(rules_config.dig(column), value) })
30
- summary_score += summary_analysis.dig(column, "tags").map { |_, tag| tag.dig("score") }.sum.to_f
30
+ summary_debt += summary_analysis.dig(column, "tags").map { |_, tag| tag.dig("debt") }.sum.to_f
31
31
  end
32
32
 
33
- [summary_analysis, summary_score]
33
+ [summary_analysis, summary_debt]
34
34
  end
35
35
 
36
36
  class << self
37
37
  private
38
38
 
39
39
  def add_summary(summary, column_name, value)
40
- summary["cardinality"] = (summary.dig("cardinality") || 1) + value.to_f if column_name.eql?("rows")
40
+ summary["cardinality"] = (summary.dig("cardinality") || 1) * value.to_f if column_name.eql?("rows")
41
41
 
42
42
  summary
43
43
  end
@@ -54,7 +54,7 @@ module QueryPolice
54
54
  next unless tag_rule.present?
55
55
 
56
56
  column_analyse["tags"].merge!(
57
- { tag => Transform.tag_rule(tag_rule).merge!({ "score" => generate_score(tag_rule, amount) }) }
57
+ { tag => Transform.tag_rule(tag_rule).merge!({ "debt" => generate_debt(tag_rule, amount) }) }
58
58
  )
59
59
  end
60
60
 
@@ -69,7 +69,7 @@ module QueryPolice
69
69
  if threshold_rule.present? && amount >= threshold_rule.dig("amount")
70
70
  return {
71
71
  "threshold" => Transform.tag_rule(threshold_rule).merge(
72
- { "amount" => amount, "score" => generate_score(threshold_rule, amount) }
72
+ { "amount" => amount, "debt" => generate_debt(threshold_rule, amount) }
73
73
  )
74
74
  }
75
75
  end
@@ -77,16 +77,16 @@ module QueryPolice
77
77
  {}
78
78
  end
79
79
 
80
- def generate_score(tag_rule, amount)
81
- score = tag_rule.dig("score", "value")
80
+ def generate_debt(tag_rule, amount)
81
+ debt = tag_rule.dig("debt", "value")
82
82
 
83
- case tag_rule.dig("score", "type").to_s
83
+ case tag_rule.dig("debt", "type").to_s
84
84
  when "base"
85
- score.to_f
85
+ debt.to_f
86
86
  when "relative"
87
- amount.to_f * score.to_f
88
- when "treshold_relative"
89
- (amount - tag_rule.dig("amount")).to_f * score.to_f
87
+ amount.to_f * debt.to_f
88
+ when "threshold_relative"
89
+ (amount - tag_rule.dig("amount")).to_f * debt.to_f
90
90
  else
91
91
  0
92
92
  end
@@ -54,13 +54,13 @@ module QueryPolice
54
54
  opts.dig("colours").present? ? impact.send(IMPACTS.dig(impact, "colour")) : impact
55
55
  end
56
56
 
57
- def score(opts)
57
+ def debt(opts)
58
58
  table, column, tag = opts.values_at("table", "column", "tag")
59
59
 
60
60
  impact = query_analytic.dig(table, "analysis", column, "tags", tag, "impact")
61
- score = query_analytic.dig(table, "analysis", column, "tags", tag, "score")
61
+ debt = query_analytic.dig(table, "analysis", column, "tags", tag, "debt")
62
62
 
63
- opts.dig("colours").present? ? score.to_s.send(IMPACTS.dig(impact, "colour")) : score
63
+ opts.dig("colours").present? ? debt.to_s.send(IMPACTS.dig(impact, "colour")) : debt
64
64
  end
65
65
 
66
66
  def table(opts)
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "analysis/dynamic_message"
4
+ require_relative "helper"
4
5
 
5
6
  module QueryPolice
6
7
  # This class is used to store analysis of a query and provide methods over them
@@ -20,7 +21,7 @@ module QueryPolice
20
21
  # "users" => {
21
22
  # "id" => 1,
22
23
  # "name" => "users",
23
- # "score" => 100.0,
24
+ # "debt" => 100.0,
24
25
  # "analysis" => {
25
26
  # "type" => {
26
27
  # "value" => all",
@@ -29,7 +30,7 @@ module QueryPolice
29
30
  # "impact" => "negative",
30
31
  # "warning" => "warning to represent the issue",
31
32
  # "suggestions" => "some follow up suggestions",
32
- # "score" => 100.0
33
+ # "debt" => 100.0
33
34
  # }
34
35
  # }
35
36
  # }
@@ -43,28 +44,28 @@ module QueryPolice
43
44
  # "amount" => 10,
44
45
  # "warning" => "warning to represent the issue",
45
46
  # "suggestions" => "some follow up suggestions",
46
- # "score" => 100.0
47
+ # "debt" => 100.0
47
48
  # }
48
49
  # }
49
- def initialize
50
+ def initialize(footer: nil)
51
+ @footer = footer || ""
52
+ @summary = {}
53
+ @summary_debt = 0
50
54
  @table_count = 0
55
+ @table_debt = 0
51
56
  @tables = {}
52
- @table_score = 0
53
- @summary = {}
54
- @summary_score = 0
55
57
  end
56
58
 
57
- attr_accessor :tables, :summary
59
+ attr_accessor :summary, :tables
58
60
 
59
61
  # register a table analysis in analysis object
60
62
  # @param name [String] name of the table
61
63
  # @param table_analysis [Hash] analysis of a table
62
- # @param score [Integer] score for that table
63
64
  # Eg.
64
65
  # {
65
66
  # "id" => 1,
66
67
  # "name" => "users",
67
- # "score" => 100.0
68
+ # "debt" => 100.0
68
69
  # "analysis" => {
69
70
  # "type" => [
70
71
  # {
@@ -72,76 +73,102 @@ module QueryPolice
72
73
  # "impact" => "negative",
73
74
  # "warning" => "warning to represent the issue",
74
75
  # "suggestions" => "some follow up suggestions",
75
- # "score" => 100.0
76
+ # "debt" => 100.0
76
77
  # }
77
78
  # ]
78
79
  # }
79
80
  # }
80
- def register_table(name, table_analysis, score)
81
+ # @param debt [Integer] debt for that table
82
+ def register_table(name, table_analysis, debt)
81
83
  @table_count += 1
82
84
  tables.merge!(
83
85
  {
84
86
  name => {
85
87
  "id" => @table_count,
86
88
  "name" => name,
87
- "score" => score,
89
+ "debt" => debt,
88
90
  "analysis" => table_analysis
89
91
  }
90
92
  }
91
93
  )
92
94
 
93
- @table_score += score
95
+ @table_debt += debt
94
96
  end
95
97
 
96
98
  # register summary based in different attributes
97
99
  # @param summary [Hash] hash of summary of analysis
98
- def register_summary(summary, score)
100
+ def register_summary(summary, debt)
99
101
  self.summary.merge!(summary)
100
- @summary_score += score
102
+ @summary_debt += debt
101
103
  end
102
104
 
103
105
  # to get analysis in pretty format with warnings and suggestions
104
- # @param opts [Hash] - possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
106
+ # @param opts [Hash] - options
107
+ # possible keys
108
+ # [
109
+ # positive: <boolean>,
110
+ # negative: <boolean>,
111
+ # caution: <boolean>,
112
+ # wrap_width: <integer>,
113
+ # skip_footer: <boolean>
114
+ # ]
105
115
  # @return [String] pretty analysis
106
- def pretty_analysis(opts)
116
+ def pretty_analysis(opts = { "negative" => true, "caution" => true })
107
117
  final_message = ""
108
118
  opts = opts.with_indifferent_access
109
119
 
110
120
  opts.slice(*IMPACTS.keys).each do |impact, value|
111
- final_message += pretty_analysis_for(impact) if value.present?
121
+ opts_ = opts.slice("wrap_width").merge({ "skip_footer" => true })
122
+ final_message += pretty_analysis_for(impact, opts_) if value.present?
112
123
  end
113
124
 
114
- final_message
125
+ opts.dig("skip_footer").present? ? final_message : final_message + @footer
115
126
  end
116
127
 
117
128
  # to get analysis in pretty format with warnings and suggestions for a impact
118
129
  # @param impact [String]
130
+ # @param opts [Hash] - options
131
+ # possible keys
132
+ # [
133
+ # wrap_width: <integer>
134
+ # skip_footer: <boolean>
135
+ # ]
119
136
  # @return [String] pretty analysis
120
- def pretty_analysis_for(impact)
121
- final_message = "query_score: #{query_score}\n\n"
137
+ def pretty_analysis_for(impact, opts = {})
138
+ final_message = ""
122
139
 
123
140
  query_analytic.each_key do |table|
124
- table_message = query_pretty_analysis(table, { impact => true })
141
+ table_message = query_pretty_analysis(table, { impact => true }.merge(opts))
125
142
 
126
143
  final_message += "#{table_message}\n" if table_message.present?
127
144
  end
128
145
 
129
- final_message
146
+ return final_message unless final_message.present?
147
+
148
+ final_message = "query_debt: #{query_debt}\n\n#{final_message}"
149
+ opts.dig("skip_footer").present? ? final_message : final_message + @footer
130
150
  end
131
151
 
132
- # to get the final score
133
- def query_score
134
- @table_score + @summary_score
152
+ # to get the final debt
153
+ def query_debt
154
+ @table_debt + @summary_debt
135
155
  end
136
156
 
137
157
  # to get analysis in pretty format with warnings and suggestions for a table
138
158
  # @param table [String] - table name
139
- # @param opts [Hash] - possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
159
+ # @param opts [Hash] - options
160
+ # possible keys
161
+ # [
162
+ # positive: <boolean>,
163
+ # negative: <boolean>,
164
+ # caution: <boolean>,
165
+ # wrap_width: <integer>
166
+ # ]
140
167
  # @return [String] pretty analysis
141
168
  def query_pretty_analysis(table, opts)
142
169
  table_analytics = Terminal::Table.new(title: table)
143
170
  table_analytics_present = false
144
- table_analytics.add_row(["score", query_analytic.dig(table, "score")])
171
+ table_analytics.add_row(["debt", query_analytic.dig(table, "debt")])
145
172
 
146
173
  opts = opts.with_indifferent_access
147
174
 
@@ -166,7 +193,7 @@ module QueryPolice
166
193
  query_analytic.dig(table, "analysis", column, "tags").each do |tag, tag_analysis|
167
194
  next unless opts.dig(tag_analysis.dig("impact")).present?
168
195
 
169
- column_analytics += tag_analytic(table, column, tag)
196
+ column_analytics += tag_analytic(table, column, tag, opts)
170
197
  end
171
198
 
172
199
  column_analytics
@@ -176,22 +203,24 @@ module QueryPolice
176
203
  tables.merge(
177
204
  "summary" => {
178
205
  "name" => "summary",
179
- "score" => @summary_score,
206
+ "debt" => @summary_debt,
180
207
  "analysis" => summary
181
208
  }
182
209
  )
183
210
  end
184
211
 
185
- def tag_analytic(table, column, tag)
212
+ def tag_analytic(table, column, tag, opts)
186
213
  tag_message = []
187
214
 
188
- opts = { "table" => table, "column" => column, "tag" => tag }
189
- message = dynamic_message(opts.merge({ "type" => "message" }))
190
- suggestion = dynamic_message(opts.merge({ "type" => "suggestion" }))
191
- tag_message << ["impact", impact(opts.merge({ "colours" => true }))]
192
- tag_message << ["tag_score", score(opts.merge({ "colours" => true }))]
193
- tag_message << ["message", message]
194
- tag_message << ["suggestion", suggestion] if suggestion.present?
215
+ variable_opts = { "table" => table, "column" => column, "tag" => tag }
216
+ message = dynamic_message(variable_opts.merge({ "type" => "message" }))
217
+ suggestion = dynamic_message(variable_opts.merge({ "type" => "suggestion" }))
218
+ wrap_width = opts.dig("wrap_width")
219
+
220
+ tag_message << ["impact", impact(variable_opts.merge({ "colours" => true }))]
221
+ tag_message << ["tag_debt", debt(variable_opts.merge({ "colours" => true }))]
222
+ tag_message << ["message", Helper.word_wrap(message, wrap_width)]
223
+ tag_message << ["suggestion", Helper.word_wrap(suggestion, wrap_width)] if suggestion.present?
195
224
 
196
225
  tag_message
197
226
  end
@@ -1,17 +1,22 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "constants"
4
+
3
5
  module QueryPolice
4
6
  # This class is used for configuration of query police
5
7
  class Config
6
- def initialize(detailed, rules_path)
7
- @detailed = detailed
8
- @rules_path = rules_path
8
+ def initialize
9
+ @action_enabled = Constants::DEFAULT_ACTION_ENABLED
10
+ @analysis_footer = Constants::DEFAULT_ANALYSIS_FOOTER
11
+ @logger_options = Constants::DEFAULT_LOGGER_OPTIONS
12
+ @rules_path = Constants::DEFAULT_RULES_PATH
13
+ @verbosity = Constants::DEFAULT_VERBOSITY
9
14
  end
10
15
 
11
- def detailed?
12
- @detailed.present?
16
+ def action_enabled?
17
+ @action_enabled.present?
13
18
  end
14
19
 
15
- attr_accessor :detailed, :rules_path
20
+ attr_accessor :action_enabled, :analysis_footer, :logger_options, :rules_path, :verbosity
16
21
  end
17
22
  end
@@ -2,13 +2,16 @@
2
2
 
3
3
  module QueryPolice
4
4
  module Constants
5
+ DEFAULT_ANALYSIS_FOOTER = ""
5
6
  DEFAULT_COLUMN_RULES = {
6
7
  "value_type" => "string"
7
8
  }.freeze
8
- DEFAULT_DETAILED = true
9
- DEFAULT_LOGGER_CONFIG = {
10
- "negative" => true
9
+ DEFAULT_ACTION_ENABLED = true
10
+ DEFAULT_LOGGER_OPTIONS = {
11
+ "negative" => true,
12
+ "caution" => true
11
13
  }.freeze
12
14
  DEFAULT_RULES_PATH = File.join(File.dirname(__FILE__), "rules.json")
15
+ DEFAULT_VERBOSITY = "detailed"
13
16
  end
14
17
  end
@@ -5,12 +5,15 @@ require_relative "helper"
5
5
  module QueryPolice
6
6
  # This module provides tools to explain queries and ActiveRecord::Relation
7
7
  module Explain
8
+ DETAILED_VERBOSITY = "detailed"
9
+
8
10
  # to get explain result in parsable format
9
11
  # @param relation [ActiveRecord::Relation, String] active record relation or raw sql query
12
+ # @param verbosity [Symbol] mode to define which EXPLAIN result should be inlcuded in final result
10
13
  # @return [Array] parsed_result - array of hashes representing EXPLAIN result for each row
11
- def full_explain(relation, detailed = true)
14
+ def full_explain(relation, verbosity = nil)
12
15
  explain_result = explain(relation)
13
- return explain_result unless detailed
16
+ return explain_result.values unless verbosity.to_s.eql?(DETAILED_VERBOSITY)
14
17
 
15
18
  detailed_explain_result = detailed_explain(relation)
16
19
 
@@ -59,10 +62,20 @@ module QueryPolice
59
62
 
60
63
  def parse_detailed_explain(explain_result)
61
64
  parsed_result = JSON.parse(explain_result&.first&.first || "{}").dig("query_block")
65
+ parsed_result = parse_detailed_explain_operations(parsed_result)
66
+
62
67
  return parsed_result.dig("nested_loop").map { |e| e.dig("table") } if parsed_result.key?("nested_loop")
63
68
 
64
69
  parsed_result.key?("table") ? [parsed_result.dig("table")] : []
65
70
  end
71
+
72
+ def parse_detailed_explain_operations(parsed_result)
73
+ parsed_result = parsed_result.dig("ordering_operation") || parsed_result
74
+ parsed_result = parsed_result.dig("grouping_operation") || parsed_result
75
+ parsed_result = parsed_result.dig("duplicates_removal") || parsed_result
76
+
77
+ parsed_result
78
+ end
66
79
  end
67
80
 
68
81
  module_function :full_explain, :explain, :detailed_explain
@@ -4,6 +4,8 @@
4
4
  module QueryPolice
5
5
  # This module define helper methods for query police
6
6
  module Helper
7
+ DEFAULT_WORD_WRAP_WIDTH = 100
8
+
7
9
  def flatten_hash(hash, prefix_key = "")
8
10
  flat_hash = {}
9
11
 
@@ -31,10 +33,31 @@ module QueryPolice
31
33
  "Please ensure that the file exists and the path is correct."
32
34
  end
33
35
 
34
- JSON.parse(File.read(rules_path))
36
+ case File.extname(rules_path)
37
+ when ".yaml", ".yml"
38
+ YAML.safe_load(File.read(rules_path))
39
+ when ".json"
40
+ JSON.parse(File.read(rules_path))
41
+ else
42
+ raise Error, "'#{File.extname(rules_path)}' extension is not supported for rules."
43
+ end
44
+ end
45
+
46
+ def word_wrap(string, width = DEFAULT_WORD_WRAP_WIDTH)
47
+ width ||= DEFAULT_WORD_WRAP_WIDTH
48
+ words = string.split
49
+ wrapped_string = ""
50
+
51
+ words.each do |word|
52
+ last_line_size = (wrapped_string.split("\n")[-1]&.size || 0)
53
+ wrapped_string = wrapped_string.strip + "\n" if (last_line_size + word.size) > width
54
+ wrapped_string += "#{word} "
55
+ end
56
+
57
+ wrapped_string.strip
35
58
  end
36
59
 
37
- module_function :flatten_hash, :logger, :load_config
60
+ module_function :flatten_hash, :logger, :load_config, :word_wrap
38
61
  end
39
62
 
40
63
  private_constant :Helper