query_police 0.1.4.beta → 0.1.5.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +408 -242
- data/examples/rules/json/absent_rule.json +13 -0
- data/examples/rules/json/basic_rule.json +22 -0
- data/examples/rules/json/complex_detailed_rule.json +18 -0
- data/examples/rules/json/threshold_rule.json +15 -0
- data/examples/rules/yaml/absent_rule.yml +11 -0
- data/examples/rules/yaml/basic_rule.yml +18 -0
- data/examples/rules/yaml/complex_detailed_rule.yml +13 -0
- data/examples/rules/yaml/threshold_rule.yml +12 -0
- data/lib/query_police/analyse.rb +16 -16
- data/lib/query_police/analysis/dynamic_message.rb +3 -3
- data/lib/query_police/analysis.rb +67 -39
- data/lib/query_police/config.rb +11 -6
- data/lib/query_police/constants.rb +6 -3
- data/lib/query_police/explain.rb +15 -2
- data/lib/query_police/helper.rb +13 -3
- data/lib/query_police/rules.json +66 -57
- data/lib/query_police/version.rb +1 -1
- data/lib/query_police.rb +58 -25
- metadata +12 -4
data/lib/query_police/rules.json
CHANGED
@@ -1,69 +1,69 @@
|
|
1
1
|
{
|
2
2
|
"select_type": {
|
3
|
-
"description": "Type of
|
3
|
+
"description": "Type of SELECT statement used in the table",
|
4
4
|
"value_type": "string",
|
5
5
|
"rules": {
|
6
6
|
"SIMPLE": {
|
7
7
|
"impact": "positive",
|
8
|
-
"message": "
|
8
|
+
"message": "This query is a simple one without any subqueries or unions",
|
9
9
|
"suggestion": ""
|
10
10
|
}
|
11
11
|
}
|
12
12
|
},
|
13
13
|
"type": {
|
14
|
-
"description": "
|
14
|
+
"description": "Type of join used in the query for a specific table",
|
15
15
|
"value_type": "string",
|
16
16
|
"rules": {
|
17
17
|
"system": {
|
18
18
|
"impact": "positive",
|
19
|
-
"message": "
|
19
|
+
"message": "This table contains zero or one rows, so no changes are needed",
|
20
20
|
"suggestion": ""
|
21
21
|
},
|
22
22
|
"const": {
|
23
23
|
"impact": "positive",
|
24
|
-
"message": "
|
24
|
+
"message": "This table has only one indexed matching row, making this the fastest type of join",
|
25
25
|
"suggestion": ""
|
26
26
|
},
|
27
27
|
"eq_ref": {
|
28
28
|
"impact": "positive",
|
29
|
-
"message": "All index parts used in join and index is
|
29
|
+
"message": "All index parts are used in this join, and the index is a primary key or unique not null",
|
30
30
|
"suggestion": ""
|
31
31
|
},
|
32
32
|
"ref": {
|
33
33
|
"impact": "caution",
|
34
|
-
"message": "All matching rows of an indexed column read for each combination of rows from previous table
|
35
|
-
"suggestion": "Ensure the referenced column is indexed and
|
34
|
+
"message": "All matching rows of an indexed column are read for each combination of rows from the previous table",
|
35
|
+
"suggestion": "Ensure the referenced column is indexed and check for null values and duplicates. If possible, consider upgrading to an eq_ref join type. You can achieve this by adding unique and not null constraints to the index - $key used in $table table"
|
36
36
|
},
|
37
37
|
"fulltext": {
|
38
38
|
"impact": "caution",
|
39
|
-
"message": "
|
40
|
-
"suggestion": "
|
39
|
+
"message": "The join uses a table FULLTEXT index, with the index key used - $key",
|
40
|
+
"suggestion": "This should only be used for columns with heavy text content"
|
41
41
|
},
|
42
42
|
"ref_or_null": {
|
43
43
|
"impact": "caution",
|
44
|
-
"message": "
|
45
|
-
"suggestion": "
|
44
|
+
"message": "A ref index is used with Null values in the $table table",
|
45
|
+
"suggestion": "Consider upgrading to an eq_ref join if possible. You can achieve this by adding unique and not null constraints to the index - $key used in $table table"
|
46
46
|
},
|
47
47
|
"index_merge": {
|
48
48
|
"impact": "caution",
|
49
|
-
"message": "
|
50
|
-
"suggestion": "
|
49
|
+
"message": "The join involves a list of indexes, with keys used: $key",
|
50
|
+
"suggestion": "Be cautious as this might be slow if the indexes are poorly chosen or if there are too many indexes being used"
|
51
51
|
},
|
52
52
|
"range": {
|
53
53
|
"impact": "caution",
|
54
|
-
"message": "
|
55
|
-
"suggestion": "Please check the range it
|
54
|
+
"message": "An index is used to find matching rows within a specific range",
|
55
|
+
"suggestion": "Please check the range; ensure it's not too broad"
|
56
56
|
},
|
57
57
|
"index": {
|
58
58
|
"impact": "caution",
|
59
|
-
"message": "
|
60
|
-
"suggestion": "
|
59
|
+
"message": "The entire index tree is scanned to find matching rows",
|
60
|
+
"suggestion": "This can be slow for large indexes (Your key length: $key_len). Use this option carefully"
|
61
61
|
},
|
62
62
|
"ALL": {
|
63
63
|
"impact": "negative",
|
64
|
-
"message": "
|
65
|
-
"suggestion": "Use index
|
66
|
-
"
|
64
|
+
"message": "The entire $table table is scanned to find matching rows. There are $amount_possible_keys possible keys that could be used",
|
65
|
+
"suggestion": "Use an index in this scenario. You can use the index from possible key: $possible_keys or add a new index to the $table table as needed",
|
66
|
+
"debt": {
|
67
67
|
"value": 100,
|
68
68
|
"type": "base"
|
69
69
|
}
|
@@ -71,27 +71,27 @@
|
|
71
71
|
}
|
72
72
|
},
|
73
73
|
"rows": {
|
74
|
-
"description": "Estimated number of rows scanned to find matching rows
|
74
|
+
"description": "Estimated number of rows scanned to find matching rows",
|
75
75
|
"value_type": "number",
|
76
76
|
"rules": {
|
77
77
|
"threshold": {
|
78
78
|
"amount": 100,
|
79
79
|
"impact": "negative",
|
80
|
-
"message": "$value rows
|
81
|
-
"suggestion": "
|
80
|
+
"message": "This query scans approximately $value rows per join for the $table table",
|
81
|
+
"suggestion": "Consider using an index from $possible_keys or adding a new index to the $table table to reduce the number of scanned rows"
|
82
82
|
}
|
83
83
|
}
|
84
84
|
},
|
85
85
|
"possible_keys": {
|
86
|
-
"description": "Index keys possible for a
|
86
|
+
"description": "Index keys possible for a specific table",
|
87
87
|
"value_type": "array",
|
88
88
|
"delimiter": ",",
|
89
89
|
"rules": {
|
90
90
|
"absent": {
|
91
91
|
"impact": "negative",
|
92
|
-
"message": "There are no possible keys for $table table
|
93
|
-
"suggestion": "
|
94
|
-
"
|
92
|
+
"message": "There are no possible keys for the $table table, which can lead to a full scan",
|
93
|
+
"suggestion": "Add appropriate index keys for the $table table",
|
94
|
+
"debt": {
|
95
95
|
"value": 50,
|
96
96
|
"type": "base"
|
97
97
|
}
|
@@ -99,24 +99,24 @@
|
|
99
99
|
"threshold": {
|
100
100
|
"amount": 5,
|
101
101
|
"impact": "negative",
|
102
|
-
"message": "There are $amount possible keys for $table table
|
103
|
-
"suggestion": "
|
104
|
-
"
|
102
|
+
"message": "There are $amount possible keys for the $table table; having too many index keys can be suboptimal",
|
103
|
+
"suggestion": "Check for unnecessary indexes in the $table table",
|
104
|
+
"debt": {
|
105
105
|
"value": 20,
|
106
|
-
"type": "
|
106
|
+
"type": "threshold_relative"
|
107
107
|
}
|
108
108
|
}
|
109
109
|
}
|
110
110
|
},
|
111
111
|
"key": {
|
112
|
-
"description": "",
|
112
|
+
"description": "The index key used",
|
113
113
|
"value_type": "string",
|
114
114
|
"rules": {
|
115
115
|
"absent": {
|
116
116
|
"impact": "negative",
|
117
|
-
"message": "
|
118
|
-
"suggestion": "
|
119
|
-
"
|
117
|
+
"message": "No index key is being used for the $table table, which may result in a full table scan",
|
118
|
+
"suggestion": "Use an index from possible_keys: $possible_keys or add a new one to the $table table as required",
|
119
|
+
"debt": {
|
120
120
|
"value": 50,
|
121
121
|
"type": "base"
|
122
122
|
}
|
@@ -124,25 +124,25 @@
|
|
124
124
|
}
|
125
125
|
},
|
126
126
|
"key_len": {
|
127
|
-
"description": "Length of the key
|
127
|
+
"description": "Length of the index key used",
|
128
128
|
"value_type": "number",
|
129
129
|
"rules": {}
|
130
130
|
},
|
131
131
|
"filtered": {
|
132
|
-
"description": "
|
132
|
+
"description": "Percentage of rows appearing from the total",
|
133
133
|
"value_type": "number",
|
134
134
|
"rules": {}
|
135
135
|
},
|
136
|
-
"
|
137
|
-
"description": "Additional information about the plan",
|
136
|
+
"Extra": {
|
137
|
+
"description": "Additional information about the query execution plan",
|
138
138
|
"value_type": "array",
|
139
139
|
"delimiter": ";",
|
140
140
|
"rules": {
|
141
141
|
"Using filesort": {
|
142
142
|
"impact": "negative",
|
143
|
-
"message": "A file-based algorithm
|
144
|
-
"suggestion": "
|
145
|
-
"
|
143
|
+
"message": "A file-based sorting algorithm is being used for your result. This could be inefficient and lead to longer query times",
|
144
|
+
"suggestion": "Ensure that the result set is small or use a proper index",
|
145
|
+
"debt": {
|
146
146
|
"value": 50,
|
147
147
|
"type": "base"
|
148
148
|
}
|
@@ -156,35 +156,44 @@
|
|
156
156
|
"impact": "",
|
157
157
|
"message": "",
|
158
158
|
"suggestion": ""
|
159
|
+
},
|
160
|
+
"no matching row in const table": {
|
161
|
+
"impact": "negative",
|
162
|
+
"message": "The query wasn't thoroughly analyzed as it refers to a constant value with no matching row in the respective tables",
|
163
|
+
"suggestion": "Add rows that correspond to the constant value or modify the reference. You can associate the reference with a constant value that has a corresponding row or refer to a column with a matching value",
|
164
|
+
"debt": {
|
165
|
+
"value": 1000,
|
166
|
+
"type": "base"
|
167
|
+
}
|
159
168
|
}
|
160
169
|
}
|
161
170
|
},
|
162
171
|
"detailed#used_columns": {
|
163
|
-
"description": "
|
172
|
+
"description": "Number of columns used to execute the query",
|
164
173
|
"value_type": "array",
|
165
174
|
"rules": {
|
166
175
|
"threshold": {
|
167
176
|
"amount": 5,
|
168
177
|
"impact": "negative",
|
169
|
-
"message": "You have selected $amount columns,
|
170
|
-
"suggestion": "Please only select required columns
|
171
|
-
"
|
178
|
+
"message": "You have selected $amount columns, which could be excessive",
|
179
|
+
"suggestion": "Please only select the required columns",
|
180
|
+
"debt": {
|
172
181
|
"value": 10,
|
173
|
-
"type": "
|
182
|
+
"type": "threshold_relative"
|
174
183
|
}
|
175
184
|
}
|
176
185
|
}
|
177
186
|
},
|
178
|
-
"detailed#cost_info#read_cost
|
179
|
-
"description": "
|
187
|
+
"detailed#cost_info#read_cost": {
|
188
|
+
"description": "Read cost to execute the query",
|
180
189
|
"value_type": "number",
|
181
190
|
"rules": {
|
182
191
|
"threshold": {
|
183
192
|
"amount": 100,
|
184
193
|
"impact": "negative",
|
185
|
-
"message": "The read cost of query is
|
186
|
-
"suggestion": "Please
|
187
|
-
"
|
194
|
+
"message": "The read cost of the query is $amount, which is considered too high. It's likely that you are scanning too many rows",
|
195
|
+
"suggestion": "Please optimize your query by using proper indexes, querying only the required data, and ensuring appropriate joins",
|
196
|
+
"debt": {
|
188
197
|
"value": 0.01,
|
189
198
|
"type": "relative"
|
190
199
|
}
|
@@ -192,15 +201,15 @@
|
|
192
201
|
}
|
193
202
|
},
|
194
203
|
"cardinality": {
|
195
|
-
"description": "
|
204
|
+
"description": "Total cardinality of the query",
|
196
205
|
"value_type": "number",
|
197
206
|
"rules": {
|
198
207
|
"threshold": {
|
199
208
|
"amount": 500,
|
200
209
|
"impact": "negative",
|
201
|
-
"message": "The cardinality of table is $amount,
|
202
|
-
"suggestion": "Please
|
203
|
-
"
|
210
|
+
"message": "The cardinality of the table is $amount, which is considered too high",
|
211
|
+
"suggestion": "Please optimize your query by using proper indexes, querying only the required data, and ensuring appropriate joins",
|
212
|
+
"debt": {
|
204
213
|
"value": 0.01,
|
205
214
|
"type": "relative"
|
206
215
|
}
|
data/lib/query_police/version.rb
CHANGED
data/lib/query_police.rb
CHANGED
@@ -23,12 +23,15 @@ module QueryPolice
|
|
23
23
|
|
24
24
|
class Error < StandardError; end
|
25
25
|
|
26
|
-
@config = Config.new
|
27
|
-
|
28
|
-
)
|
26
|
+
@config = Config.new
|
27
|
+
@actions = []
|
29
28
|
|
30
29
|
CONFIG_METHODS = %i[
|
31
|
-
|
30
|
+
action_enabled action_enabled? action_enabled=
|
31
|
+
analysis_footer analysis_footer=
|
32
|
+
logger_options logger_options=
|
33
|
+
rules_path rules_path=
|
34
|
+
verbosity verbosity=
|
32
35
|
].freeze
|
33
36
|
|
34
37
|
def_delegators :config, *CONFIG_METHODS
|
@@ -38,15 +41,15 @@ module QueryPolice
|
|
38
41
|
# @return [QueryPolice::Analysis] analysis - contains the analysis of the query
|
39
42
|
def analyse(relation)
|
40
43
|
rules_config = Helper.load_config(config.rules_path)
|
41
|
-
analysis = Analysis.new
|
44
|
+
analysis = Analysis.new(footer: config.analysis_footer)
|
42
45
|
summary = {}
|
43
46
|
|
44
|
-
query_plan = Explain.full_explain(relation, config.
|
47
|
+
query_plan = Explain.full_explain(relation, config.verbosity)
|
45
48
|
|
46
49
|
query_plan.each do |table|
|
47
|
-
table_analysis, summary,
|
50
|
+
table_analysis, summary, table_debt = Analyse.table(table, summary, rules_config)
|
48
51
|
|
49
|
-
analysis.register_table(table.dig("table"), table_analysis,
|
52
|
+
analysis.register_table(table.dig("table"), table_analysis, table_debt)
|
50
53
|
end
|
51
54
|
|
52
55
|
analysis.register_summary(*Analyse.generate_summary(rules_config, summary))
|
@@ -54,28 +57,58 @@ module QueryPolice
|
|
54
57
|
analysis
|
55
58
|
end
|
56
59
|
|
57
|
-
|
58
|
-
# @param silent [Boolean] silent errors for logger
|
59
|
-
# @param logger_config [Hash] possible options [positive: <boolean>, negative: <boolean>, caution: <boolean>]
|
60
|
-
def subscribe_logger(silent: false, logger_config: Constants::DEFAULT_LOGGER_CONFIG)
|
61
|
-
ActiveSupport::Notifications.subscribe("sql.active_record") do |_, _, _, _, payload|
|
62
|
-
begin
|
63
|
-
if !payload[:exception].present? && payload[:name] =~ /.* Load/
|
64
|
-
analysis = analyse(payload[:sql])
|
60
|
+
module_function :analyse, *CONFIG_METHODS
|
65
61
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
62
|
+
class << self
|
63
|
+
attr_accessor :config
|
64
|
+
|
65
|
+
def add_action(&block)
|
66
|
+
@actions << block
|
67
|
+
|
68
|
+
true
|
69
|
+
end
|
70
70
|
|
71
|
-
|
71
|
+
def configure
|
72
|
+
yield(config)
|
73
|
+
end
|
74
|
+
|
75
|
+
def evade_actions
|
76
|
+
old_config_value = config.action_enabled
|
77
|
+
config.action_enabled = false
|
78
|
+
|
79
|
+
return_value = yield
|
80
|
+
|
81
|
+
config.action_enabled = old_config_value
|
82
|
+
return_value
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
# perform actions on the analysis of a query
|
88
|
+
# @param query [ActiveRecord::Relation, String]
|
89
|
+
def perform_actions(query)
|
90
|
+
return unless config.action_enabled?
|
91
|
+
|
92
|
+
analysis = analyse(query)
|
93
|
+
Helper.logger(analysis.pretty_analysis(config.logger_options))
|
94
|
+
|
95
|
+
@actions.each do |action|
|
96
|
+
action.call(analysis)
|
72
97
|
end
|
73
98
|
end
|
74
|
-
end
|
75
99
|
|
76
|
-
|
77
|
-
|
100
|
+
# to subscribe to active support notification to perform actions after each query
|
101
|
+
def subscribe_action
|
102
|
+
ActiveSupport::Notifications.subscribe("sql.active_record") do |_, _, _, _, payload|
|
103
|
+
begin
|
104
|
+
perform_actions(payload[:sql]) if !payload[:exception].present? && payload[:name] =~ /.* Load/
|
105
|
+
rescue StandardError => e
|
106
|
+
Helper.logger("#{name}::#{e.class}: #{e.message}", "error")
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
78
110
|
end
|
79
111
|
|
80
|
-
|
112
|
+
# subscribe to active support notification on module usage
|
113
|
+
subscribe_action
|
81
114
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: query_police
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- strikeraryu
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-09-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -76,7 +76,7 @@ dependencies:
|
|
76
76
|
requirements:
|
77
77
|
- - ">="
|
78
78
|
- !ruby/object:Gem::Version
|
79
|
-
version:
|
79
|
+
version: 1.5.0
|
80
80
|
- - "<"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: 3.0.2
|
@@ -86,7 +86,7 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
89
|
+
version: 1.5.0
|
90
90
|
- - "<"
|
91
91
|
- !ruby/object:Gem::Version
|
92
92
|
version: 3.0.2
|
@@ -105,6 +105,14 @@ files:
|
|
105
105
|
- LICENSE.txt
|
106
106
|
- README.md
|
107
107
|
- Rakefile
|
108
|
+
- examples/rules/json/absent_rule.json
|
109
|
+
- examples/rules/json/basic_rule.json
|
110
|
+
- examples/rules/json/complex_detailed_rule.json
|
111
|
+
- examples/rules/json/threshold_rule.json
|
112
|
+
- examples/rules/yaml/absent_rule.yml
|
113
|
+
- examples/rules/yaml/basic_rule.yml
|
114
|
+
- examples/rules/yaml/complex_detailed_rule.yml
|
115
|
+
- examples/rules/yaml/threshold_rule.yml
|
108
116
|
- lib/query_police.rb
|
109
117
|
- lib/query_police/analyse.rb
|
110
118
|
- lib/query_police/analysis.rb
|