pgdexter 0.5.6 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/LICENSE.txt +1 -1
- data/README.md +4 -14
- data/lib/dexter/client.rb +46 -29
- data/lib/dexter/collector.rb +11 -19
- data/lib/dexter/column_resolver.rb +74 -0
- data/lib/dexter/connection.rb +96 -0
- data/lib/dexter/index_creator.rb +72 -0
- data/lib/dexter/indexer.rb +175 -423
- data/lib/dexter/logging.rb +1 -1
- data/lib/dexter/parsers/csv_log_parser.rb +25 -0
- data/lib/dexter/{json_log_parser.rb → parsers/json_log_parser.rb} +5 -3
- data/lib/dexter/{log_parser.rb → parsers/log_parser.rb} +1 -8
- data/lib/dexter/{sql_log_parser.rb → parsers/sql_log_parser.rb} +3 -2
- data/lib/dexter/{stderr_log_parser.rb → parsers/stderr_log_parser.rb} +4 -8
- data/lib/dexter/processor.rb +10 -24
- data/lib/dexter/query.rb +14 -31
- data/lib/dexter/sources/log_source.rb +26 -0
- data/lib/dexter/{pg_stat_activity_parser.rb → sources/pg_stat_activity_source.rb} +10 -6
- data/lib/dexter/sources/pg_stat_statements_source.rb +34 -0
- data/lib/dexter/sources/statement_source.rb +11 -0
- data/lib/dexter/table_resolver.rb +120 -0
- data/lib/dexter/version.rb +1 -1
- data/lib/dexter.rb +16 -7
- data/lib/pgdexter.rb +1 -0
- metadata +20 -12
- data/lib/dexter/csv_log_parser.rb +0 -24
data/lib/dexter/indexer.rb
CHANGED
@@ -2,77 +2,77 @@ module Dexter
|
|
2
2
|
class Indexer
|
3
3
|
include Logging
|
4
4
|
|
5
|
-
def initialize(options)
|
5
|
+
def initialize(connection:, **options)
|
6
|
+
@connection = connection
|
6
7
|
@create = options[:create]
|
7
8
|
@tablespace = options[:tablespace]
|
8
9
|
@log_level = options[:log_level]
|
9
10
|
@exclude_tables = options[:exclude]
|
10
11
|
@include_tables = Array(options[:include].split(",")) if options[:include]
|
11
|
-
@log_sql = options[:log_sql]
|
12
12
|
@log_explain = options[:log_explain]
|
13
|
-
@min_time = options[:min_time] || 0
|
14
|
-
@min_calls = options[:min_calls] || 0
|
15
13
|
@analyze = options[:analyze]
|
14
|
+
@min_cost = options[:min_cost].to_i
|
16
15
|
@min_cost_savings_pct = options[:min_cost_savings_pct].to_i
|
17
16
|
@options = options
|
18
|
-
@
|
19
|
-
|
20
|
-
if server_version_num < 110000
|
21
|
-
raise Dexter::Abort, "This version of Dexter requires Postgres 11+"
|
22
|
-
end
|
23
|
-
|
24
|
-
check_extension
|
25
|
-
|
26
|
-
execute("SET lock_timeout = '5s'")
|
27
|
-
end
|
28
|
-
|
29
|
-
def process_stat_statements
|
30
|
-
queries = stat_statements.map { |q| Query.new(q) }.sort_by(&:fingerprint).group_by(&:fingerprint).map { |_, v| v.first }
|
31
|
-
log "Processing #{queries.size} new query fingerprints"
|
32
|
-
process_queries(queries)
|
17
|
+
@server_version_num = @connection.server_version_num
|
33
18
|
end
|
34
19
|
|
20
|
+
# TODO recheck server version?
|
35
21
|
def process_queries(queries)
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
tables = Set.new(database_tables + materialized_views)
|
22
|
+
TableResolver.new(@connection, queries, log_level: @log_level).perform
|
23
|
+
candidate_queries = queries.reject(&:missing_tables)
|
40
24
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
tables.group_by { |t| t.split(".")[-1] }.each do |group, t2|
|
45
|
-
no_schema_tables[group] = t2.sort_by { |t| [search_path_index[t.split(".")[0]] || 1000000, t] }[0]
|
25
|
+
tables = determine_tables(candidate_queries)
|
26
|
+
candidate_queries.each do |query|
|
27
|
+
query.candidate_tables = query.tables.select { |t| tables.include?(t) }.sort
|
46
28
|
end
|
29
|
+
candidate_queries.select! { |q| q.candidate_tables.any? }
|
47
30
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
31
|
+
if tables.any?
|
32
|
+
# analyze tables if needed
|
33
|
+
# TODO remove @log_level in 0.7.0
|
34
|
+
analyze_tables(tables) if @analyze || @log_level == "debug2"
|
35
|
+
|
36
|
+
# get initial costs for queries
|
37
|
+
reset_hypothetical_indexes
|
38
|
+
calculate_plan(candidate_queries)
|
39
|
+
candidate_queries.select! { |q| q.initial_cost && q.initial_cost >= @min_cost }
|
40
|
+
|
41
|
+
# find columns
|
42
|
+
ColumnResolver.new(@connection, candidate_queries, log_level: @log_level).perform
|
43
|
+
candidate_queries.each do |query|
|
44
|
+
# no reason to use btree index for json columns
|
45
|
+
# TODO check type supports btree
|
46
|
+
query.candidate_columns = query.columns.reject { |c| ["json", "jsonb", "point"].include?(c[:type]) }.sort_by { |c| [c[:table], c[:column]] }
|
47
|
+
end
|
48
|
+
candidate_queries.select! { |q| q.candidate_columns.any? }
|
53
49
|
|
54
|
-
|
55
|
-
|
56
|
-
view_tables.each do |v, vt|
|
57
|
-
view_tables[v] = vt.flat_map { |t| view_tables[t] || [t] }.uniq
|
50
|
+
# create hypothetical indexes and explain queries
|
51
|
+
batch_hypothetical_indexes(candidate_queries)
|
58
52
|
end
|
59
53
|
|
60
|
-
#
|
61
|
-
queries
|
62
|
-
# add schema to table if needed
|
63
|
-
query.tables = query.tables.map { |t| no_schema_tables[t] || t }
|
54
|
+
# see if new indexes were used and meet bar
|
55
|
+
new_indexes = determine_indexes(queries, tables)
|
64
56
|
|
65
|
-
|
66
|
-
|
67
|
-
query.tables_from_views = new_tables - query.tables
|
68
|
-
query.tables = new_tables
|
57
|
+
# display new indexes
|
58
|
+
show_new_indexes(new_indexes)
|
69
59
|
|
70
|
-
|
71
|
-
|
72
|
-
|
60
|
+
# display debug info
|
61
|
+
show_debug_info(new_indexes, queries) if @log_level.start_with?("debug")
|
62
|
+
|
63
|
+
# create new indexes
|
64
|
+
IndexCreator.new(@connection, self, new_indexes, @tablespace).perform if @create && new_indexes.any?
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
73
68
|
|
69
|
+
def reset_hypothetical_indexes
|
70
|
+
execute("SELECT hypopg_reset()")
|
71
|
+
end
|
72
|
+
|
73
|
+
def determine_tables(candidate_queries)
|
74
74
|
# set tables
|
75
|
-
tables = Set.new(
|
75
|
+
tables = Set.new(candidate_queries.flat_map(&:tables))
|
76
76
|
|
77
77
|
# must come after missing tables set
|
78
78
|
if @include_tables
|
@@ -88,54 +88,10 @@ module Dexter
|
|
88
88
|
# remove system tables
|
89
89
|
tables.delete_if { |t| t.start_with?("information_schema.", "pg_catalog.") }
|
90
90
|
|
91
|
-
|
92
|
-
query.candidate_tables = !query.missing_tables && query.tables.any? { |t| tables.include?(t) }
|
93
|
-
end
|
94
|
-
|
95
|
-
# analyze tables if needed
|
96
|
-
analyze_tables(tables) if tables.any? && (@analyze || @log_level == "debug2")
|
97
|
-
|
98
|
-
# create hypothetical indexes and explain queries
|
99
|
-
if tables.any?
|
100
|
-
# process in batches to prevent "hypopg: not more oid available" error
|
101
|
-
# https://hypopg.readthedocs.io/en/rel1_stable/usage.html#configuration
|
102
|
-
queries.select(&:candidate_tables).each_slice(500) do |batch|
|
103
|
-
create_hypothetical_indexes(batch)
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
# see if new indexes were used and meet bar
|
108
|
-
new_indexes = determine_indexes(queries, tables)
|
109
|
-
|
110
|
-
# display and create new indexes
|
111
|
-
show_and_create_indexes(new_indexes, queries)
|
112
|
-
end
|
113
|
-
|
114
|
-
private
|
115
|
-
|
116
|
-
def check_extension
|
117
|
-
extension = execute("SELECT installed_version FROM pg_available_extensions WHERE name = 'hypopg'").first
|
118
|
-
|
119
|
-
if extension.nil?
|
120
|
-
raise Dexter::Abort, "Install HypoPG first: https://github.com/ankane/dexter#installation"
|
121
|
-
end
|
122
|
-
|
123
|
-
if extension["installed_version"].nil?
|
124
|
-
if @options[:enable_hypopg]
|
125
|
-
execute("CREATE EXTENSION hypopg")
|
126
|
-
else
|
127
|
-
raise Dexter::Abort, "Run `CREATE EXTENSION hypopg` or pass --enable-hypopg"
|
128
|
-
end
|
129
|
-
end
|
91
|
+
tables
|
130
92
|
end
|
131
93
|
|
132
|
-
def
|
133
|
-
execute("SELECT hypopg_reset()")
|
134
|
-
end
|
135
|
-
|
136
|
-
def analyze_tables(tables)
|
137
|
-
tables = tables.to_a.sort
|
138
|
-
|
94
|
+
def analyze_stats(tables)
|
139
95
|
query = <<~SQL
|
140
96
|
SELECT
|
141
97
|
schemaname || '.' || relname AS table,
|
@@ -146,10 +102,14 @@ module Dexter
|
|
146
102
|
WHERE
|
147
103
|
schemaname || '.' || relname IN (#{tables.size.times.map { |i| "$#{i + 1}" }.join(", ")})
|
148
104
|
SQL
|
149
|
-
|
105
|
+
execute(query, params: tables)
|
106
|
+
end
|
107
|
+
|
108
|
+
def analyze_tables(tables)
|
109
|
+
tables = tables.to_a.sort
|
150
110
|
|
151
111
|
last_analyzed = {}
|
152
|
-
analyze_stats.each do |stats|
|
112
|
+
analyze_stats(tables).each do |stats|
|
153
113
|
last_analyzed[stats["table"]] = Time.parse(stats["last_analyze"]) if stats["last_analyze"]
|
154
114
|
end
|
155
115
|
|
@@ -162,7 +122,7 @@ module Dexter
|
|
162
122
|
end
|
163
123
|
|
164
124
|
if @analyze && (!la || la < Time.now - 3600)
|
165
|
-
statement = "ANALYZE #{quote_ident(table)}"
|
125
|
+
statement = "ANALYZE #{@connection.quote_ident(table)}"
|
166
126
|
log "Running analyze: #{statement}"
|
167
127
|
execute(statement)
|
168
128
|
end
|
@@ -186,71 +146,83 @@ module Dexter
|
|
186
146
|
end
|
187
147
|
end
|
188
148
|
|
189
|
-
|
190
|
-
|
149
|
+
# process in batches to prevent "hypopg: not more oid available" error
|
150
|
+
# https://hypopg.readthedocs.io/en/rel1_stable/usage.html#configuration
|
151
|
+
def batch_hypothetical_indexes(candidate_queries)
|
152
|
+
batch_count = 0
|
153
|
+
batch = []
|
154
|
+
single_column_indexes = Set.new
|
155
|
+
multicolumn_indexes = Set.new
|
191
156
|
|
192
|
-
|
157
|
+
# sort to improve batching
|
158
|
+
# TODO improve
|
159
|
+
candidate_queries.sort_by! { |q| q.candidate_columns.map { |c| [c[:table], c[:column]] } }
|
193
160
|
|
194
|
-
|
195
|
-
|
196
|
-
explainable_queries = queries.select { |q| q.plans.any? && q.high_cost? }
|
161
|
+
candidate_queries.each do |query|
|
162
|
+
batch << query
|
197
163
|
|
198
|
-
|
199
|
-
tables = Set.new(explainable_queries.flat_map(&:tables))
|
200
|
-
tables_from_views = Set.new(explainable_queries.flat_map(&:tables_from_views))
|
164
|
+
single_column_indexes.merge(query.candidate_columns)
|
201
165
|
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
explainable_queries.each do |query|
|
207
|
-
log "Finding columns: #{query.statement}" if @log_level == "debug3"
|
208
|
-
begin
|
209
|
-
find_columns(query.tree).each do |col|
|
210
|
-
last_col = col["fields"].last
|
211
|
-
if last_col["String"]
|
212
|
-
possible_columns << last_col["String"]["sval"]
|
213
|
-
end
|
214
|
-
end
|
215
|
-
rescue JSON::NestingError
|
216
|
-
if @log_level.start_with?("debug")
|
217
|
-
log colorize("ERROR: Cannot get columns", :red)
|
218
|
-
end
|
219
|
-
end
|
166
|
+
# TODO for multicolumn indexes, use ordering
|
167
|
+
columns_by_table = query.candidate_columns.group_by { |c| c[:table] }
|
168
|
+
columns_by_table.each do |_, columns|
|
169
|
+
multicolumn_indexes.merge(columns.permutation(2).to_a)
|
220
170
|
end
|
221
171
|
|
222
|
-
|
223
|
-
|
224
|
-
|
172
|
+
if single_column_indexes.size + multicolumn_indexes.size >= 500
|
173
|
+
create_hypothetical_indexes(batch, single_column_indexes, multicolumn_indexes, batch_count)
|
174
|
+
batch_count += 1
|
175
|
+
batch.clear
|
176
|
+
single_column_indexes.clear
|
177
|
+
multicolumn_indexes.clear
|
178
|
+
end
|
179
|
+
end
|
225
180
|
|
226
|
-
|
227
|
-
|
181
|
+
if batch.any?
|
182
|
+
create_hypothetical_indexes(batch, single_column_indexes, multicolumn_indexes, batch_count)
|
183
|
+
end
|
184
|
+
end
|
228
185
|
|
229
|
-
|
230
|
-
|
186
|
+
def create_candidate_indexes(candidate_indexes, index_mapping)
|
187
|
+
candidate_indexes.each do |columns|
|
188
|
+
begin
|
189
|
+
index_name = create_hypothetical_index(columns[0][:table], columns.map { |c| c[:column] })
|
190
|
+
index_mapping[index_name] = columns
|
191
|
+
rescue PG::UndefinedObject
|
192
|
+
# data type x has no default operator class for access method "btree"
|
193
|
+
end
|
194
|
+
end
|
195
|
+
rescue PG::InternalError
|
196
|
+
# hypopg: not more oid available
|
197
|
+
log colorize("WARNING: Limiting index candidates", :yellow) if @log_level == "debug2"
|
198
|
+
end
|
231
199
|
|
232
|
-
|
233
|
-
|
200
|
+
def create_hypothetical_indexes(queries, single_column_indexes, multicolumn_indexes, batch_count)
|
201
|
+
index_mapping = {}
|
202
|
+
reset_hypothetical_indexes
|
234
203
|
|
235
|
-
|
236
|
-
|
237
|
-
|
204
|
+
# check single column indexes
|
205
|
+
create_candidate_indexes(single_column_indexes.map { |c| [c] }, index_mapping)
|
206
|
+
calculate_plan(queries)
|
207
|
+
|
208
|
+
# check multicolumn indexes
|
209
|
+
create_candidate_indexes(multicolumn_indexes, index_mapping)
|
210
|
+
calculate_plan(queries)
|
238
211
|
|
212
|
+
# save index mapping for analysis
|
239
213
|
queries.each do |query|
|
240
|
-
query.
|
214
|
+
query.index_mapping = index_mapping
|
241
215
|
end
|
242
|
-
end
|
243
216
|
|
244
|
-
|
245
|
-
|
246
|
-
find_by_key(plan, "ColumnRef")
|
217
|
+
# TODO different log level?
|
218
|
+
log "Batch #{batch_count + 1}: #{queries.size} queries, #{index_mapping.size} hypothetical indexes" if @log_level == "debug2"
|
247
219
|
end
|
248
220
|
|
249
221
|
def find_indexes(plan)
|
250
|
-
find_by_key(plan, "Index Name")
|
222
|
+
self.class.find_by_key(plan, "Index Name")
|
251
223
|
end
|
252
224
|
|
253
|
-
def find_by_key(plan, key)
|
225
|
+
def self.find_by_key(plan, key)
|
254
226
|
result = []
|
255
227
|
queue = [plan]
|
256
228
|
while queue.any?
|
@@ -271,16 +243,16 @@ module Dexter
|
|
271
243
|
result
|
272
244
|
end
|
273
245
|
|
274
|
-
def hypo_indexes_from_plan(
|
246
|
+
def hypo_indexes_from_plan(index_mapping, plan, index_set)
|
275
247
|
query_indexes = []
|
276
248
|
|
277
249
|
find_indexes(plan).uniq.sort.each do |index_name|
|
278
|
-
|
250
|
+
columns = index_mapping[index_name]
|
279
251
|
|
280
|
-
if
|
252
|
+
if columns
|
281
253
|
index = {
|
282
|
-
table:
|
283
|
-
columns:
|
254
|
+
table: columns[0][:table],
|
255
|
+
columns: columns.map { |c| c[:column] }
|
284
256
|
}
|
285
257
|
|
286
258
|
unless index_set.include?([index[:table], index[:columns]])
|
@@ -310,10 +282,10 @@ module Dexter
|
|
310
282
|
end
|
311
283
|
end
|
312
284
|
|
313
|
-
savings_ratio =
|
285
|
+
savings_ratio = 1 - @min_cost_savings_pct / 100.0
|
314
286
|
|
315
287
|
queries.each do |query|
|
316
|
-
if query.
|
288
|
+
if query.fully_analyzed?
|
317
289
|
new_cost, new_cost2 = query.costs[1..2]
|
318
290
|
|
319
291
|
cost_savings = new_cost < query.initial_cost * savings_ratio
|
@@ -322,11 +294,11 @@ module Dexter
|
|
322
294
|
cost_savings2 = new_cost > 100 && new_cost2 < new_cost * savings_ratio
|
323
295
|
|
324
296
|
key = cost_savings2 ? 2 : 1
|
325
|
-
query_indexes = hypo_indexes_from_plan(query.
|
297
|
+
query_indexes = hypo_indexes_from_plan(query.index_mapping, query.plans[key], index_set)
|
326
298
|
|
327
299
|
# likely a bad suggestion, so try single column
|
328
300
|
if cost_savings2 && query_indexes.size > 1
|
329
|
-
query_indexes = hypo_indexes_from_plan(query.
|
301
|
+
query_indexes = hypo_indexes_from_plan(query.index_mapping, query.plans[1], index_set)
|
330
302
|
cost_savings2 = false
|
331
303
|
end
|
332
304
|
|
@@ -348,7 +320,7 @@ module Dexter
|
|
348
320
|
|
349
321
|
query_indexes.each do |query_index|
|
350
322
|
reset_hypothetical_indexes
|
351
|
-
create_hypothetical_index(query_index[:table], query_index[:columns]
|
323
|
+
create_hypothetical_index(query_index[:table], query_index[:columns])
|
352
324
|
plan3 = plan(query.statement)
|
353
325
|
cost3 = plan3["Total Cost"]
|
354
326
|
|
@@ -399,8 +371,8 @@ module Dexter
|
|
399
371
|
|
400
372
|
# TODO optimize
|
401
373
|
if @log_level.start_with?("debug")
|
402
|
-
query.pass1_indexes = hypo_indexes_from_plan(query.
|
403
|
-
query.pass2_indexes = hypo_indexes_from_plan(query.
|
374
|
+
query.pass1_indexes = hypo_indexes_from_plan(query.index_mapping, query.plans[1], index_set)
|
375
|
+
query.pass2_indexes = hypo_indexes_from_plan(query.index_mapping, query.plans[2], index_set)
|
404
376
|
end
|
405
377
|
end
|
406
378
|
end
|
@@ -424,8 +396,7 @@ module Dexter
|
|
424
396
|
end
|
425
397
|
end
|
426
398
|
|
427
|
-
def
|
428
|
-
# print summary
|
399
|
+
def show_new_indexes(new_indexes)
|
429
400
|
if new_indexes.any?
|
430
401
|
new_indexes.each do |index|
|
431
402
|
log colorize("Index found: #{index[:table]} (#{index[:columns].join(", ")})", :green)
|
@@ -433,121 +404,56 @@ module Dexter
|
|
433
404
|
else
|
434
405
|
log "No new indexes found"
|
435
406
|
end
|
407
|
+
end
|
436
408
|
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
log "Need #{@min_cost_savings_pct}% cost savings to suggest index"
|
470
|
-
end
|
471
|
-
else
|
472
|
-
log "Could not run explain"
|
409
|
+
def show_debug_info(new_indexes, queries)
|
410
|
+
index_queries = new_indexes.flat_map { |i| i[:queries].sort_by(&:fingerprint) }
|
411
|
+
if @log_level == "debug2"
|
412
|
+
fingerprints = Set.new(index_queries.map(&:fingerprint))
|
413
|
+
index_queries.concat(queries.reject { |q| fingerprints.include?(q.fingerprint) }.sort_by(&:fingerprint))
|
414
|
+
end
|
415
|
+
index_queries.each do |query|
|
416
|
+
log "-" * 80
|
417
|
+
log "Query #{query.fingerprint}"
|
418
|
+
log "Total time: #{(query.total_time / 60000.0).round(1)} min, avg time: #{(query.total_time / query.calls.to_f).round} ms, calls: #{query.calls}" if query.calls > 0
|
419
|
+
|
420
|
+
if query.fingerprint == "unknown"
|
421
|
+
log "Could not parse query"
|
422
|
+
elsif query.tables.empty?
|
423
|
+
log "No tables"
|
424
|
+
elsif query.missing_tables
|
425
|
+
log "Tables not present in current database"
|
426
|
+
elsif query.candidate_tables.empty?
|
427
|
+
log "No candidate tables for indexes"
|
428
|
+
elsif !query.initial_cost
|
429
|
+
log "Could not run explain"
|
430
|
+
elsif query.initial_cost < @min_cost
|
431
|
+
log "Low initial cost: #{query.initial_cost}"
|
432
|
+
elsif query.candidate_columns.empty?
|
433
|
+
log "No candidate columns for indexes"
|
434
|
+
elsif query.fully_analyzed?
|
435
|
+
query_indexes = query.indexes || []
|
436
|
+
log "Start: #{query.costs[0]}"
|
437
|
+
log "Pass1: #{query.costs[1]} : #{log_indexes(query.pass1_indexes || [])}"
|
438
|
+
log "Pass2: #{query.costs[2]} : #{log_indexes(query.pass2_indexes || [])}"
|
439
|
+
if query.costs[3]
|
440
|
+
log "Pass3: #{query.costs[3]} : #{log_indexes(query.pass3_indexes || [])}"
|
473
441
|
end
|
474
|
-
log
|
475
|
-
|
476
|
-
|
477
|
-
end
|
478
|
-
end
|
479
|
-
|
480
|
-
# create
|
481
|
-
if @create && new_indexes.any?
|
482
|
-
# 1. create lock
|
483
|
-
# 2. refresh existing index list
|
484
|
-
# 3. create indexes that still don't exist
|
485
|
-
# 4. release lock
|
486
|
-
with_advisory_lock do
|
487
|
-
new_indexes.each do |index|
|
488
|
-
unless index_exists?(index)
|
489
|
-
statement = String.new("CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})")
|
490
|
-
statement << " TABLESPACE #{quote_ident(@tablespace)}" if @tablespace
|
491
|
-
log "Creating index: #{statement}"
|
492
|
-
started_at = monotonic_time
|
493
|
-
begin
|
494
|
-
execute(statement)
|
495
|
-
log "Index created: #{((monotonic_time - started_at) * 1000).to_i} ms"
|
496
|
-
rescue PG::LockNotAvailable
|
497
|
-
log "Could not acquire lock: #{index[:table]}"
|
498
|
-
end
|
499
|
-
end
|
442
|
+
log "Final: #{query.new_cost} : #{log_indexes(query.suggest_index ? query_indexes : [])}"
|
443
|
+
if (query.pass1_indexes.any? || query.pass2_indexes.any?) && !query.suggest_index
|
444
|
+
log "Need #{@min_cost_savings_pct}% cost savings to suggest index"
|
500
445
|
end
|
501
|
-
end
|
502
|
-
end
|
503
|
-
|
504
|
-
new_indexes
|
505
|
-
end
|
506
|
-
|
507
|
-
def monotonic_time
|
508
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
509
|
-
end
|
510
|
-
|
511
|
-
def conn
|
512
|
-
@conn ||= begin
|
513
|
-
# set connect timeout if none set
|
514
|
-
ENV["PGCONNECT_TIMEOUT"] ||= "3"
|
515
|
-
|
516
|
-
if @options[:dbname].start_with?("postgres://", "postgresql://")
|
517
|
-
config = @options[:dbname]
|
518
446
|
else
|
519
|
-
|
520
|
-
host: @options[:host],
|
521
|
-
port: @options[:port],
|
522
|
-
dbname: @options[:dbname],
|
523
|
-
user: @options[:username]
|
524
|
-
}.reject { |_, value| value.to_s.empty? }
|
525
|
-
config = config[:dbname] if config.keys == [:dbname] && config[:dbname].include?("=")
|
447
|
+
log "Could not run explain"
|
526
448
|
end
|
527
|
-
|
449
|
+
log
|
450
|
+
log query.statement
|
451
|
+
log
|
528
452
|
end
|
529
|
-
rescue PG::ConnectionBad => e
|
530
|
-
raise Dexter::Abort, e.message
|
531
453
|
end
|
532
454
|
|
533
|
-
def execute(
|
534
|
-
|
535
|
-
#
|
536
|
-
# Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
|
537
|
-
# (There can be semicolons in it, but not more than one nonempty command.)
|
538
|
-
# This is a limitation of the underlying protocol, but has some usefulness
|
539
|
-
# as an extra defense against SQL-injection attacks.
|
540
|
-
# https://www.postgresql.org/docs/current/static/libpq-exec.html
|
541
|
-
query = squish(query) if pretty
|
542
|
-
log colorize("[sql] #{query}#{params.any? ? " /*#{params.to_json}*/" : ""}", :cyan) if @log_sql
|
543
|
-
|
544
|
-
@mutex.synchronize do
|
545
|
-
if use_exec
|
546
|
-
conn.exec("#{query} /*dexter*/").to_a
|
547
|
-
else
|
548
|
-
conn.exec_params("#{query} /*dexter*/", params).to_a
|
549
|
-
end
|
550
|
-
end
|
455
|
+
def execute(...)
|
456
|
+
@connection.execute(...)
|
551
457
|
end
|
552
458
|
|
553
459
|
def plan(query)
|
@@ -557,7 +463,7 @@ module Dexter
|
|
557
463
|
# try to EXPLAIN normalized queries
|
558
464
|
# https://dev.to/yugabyte/explain-from-pgstatstatements-normalized-queries-how-to-always-get-the-generic-plan-in--5cfi
|
559
465
|
normalized = query.include?("$1")
|
560
|
-
generic_plan = normalized && server_version_num >= 160000
|
466
|
+
generic_plan = normalized && @server_version_num >= 160000
|
561
467
|
explain_normalized = normalized && !generic_plan
|
562
468
|
if explain_normalized
|
563
469
|
prepared_name = "dexter_prepared"
|
@@ -569,16 +475,7 @@ module Dexter
|
|
569
475
|
execute("BEGIN")
|
570
476
|
transaction = true
|
571
477
|
|
572
|
-
|
573
|
-
execute("SET LOCAL plan_cache_mode = force_generic_plan")
|
574
|
-
else
|
575
|
-
execute("SET LOCAL cpu_operator_cost = 1e42")
|
576
|
-
5.times do
|
577
|
-
execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}", pretty: false)
|
578
|
-
end
|
579
|
-
execute("ROLLBACK")
|
580
|
-
execute("BEGIN")
|
581
|
-
end
|
478
|
+
execute("SET LOCAL plan_cache_mode = force_generic_plan")
|
582
479
|
end
|
583
480
|
|
584
481
|
explain_prefix = generic_plan ? "GENERIC_PLAN, " : ""
|
@@ -599,140 +496,8 @@ module Dexter
|
|
599
496
|
end
|
600
497
|
end
|
601
498
|
|
602
|
-
|
603
|
-
|
604
|
-
columns_by_table.each do |table, cols|
|
605
|
-
# no reason to use btree index for json columns
|
606
|
-
cols.reject { |c| ["json", "jsonb"].include?(c[:type]) }.permutation(n) do |col_set|
|
607
|
-
index_name = create_hypothetical_index(table, col_set)
|
608
|
-
candidates[index_name] = col_set
|
609
|
-
end
|
610
|
-
end
|
611
|
-
end
|
612
|
-
|
613
|
-
def create_hypothetical_index(table, col_set)
|
614
|
-
execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(table)} (#{col_set.map { |c| quote_ident(c[:column]) }.join(", ")})')").first["indexname"]
|
615
|
-
end
|
616
|
-
|
617
|
-
def database_tables
|
618
|
-
result = execute <<~SQL
|
619
|
-
SELECT
|
620
|
-
table_schema || '.' || table_name AS table_name
|
621
|
-
FROM
|
622
|
-
information_schema.tables
|
623
|
-
WHERE
|
624
|
-
table_catalog = current_database()
|
625
|
-
AND table_type IN ('BASE TABLE', 'VIEW')
|
626
|
-
SQL
|
627
|
-
result.map { |r| r["table_name"] }
|
628
|
-
end
|
629
|
-
|
630
|
-
def materialized_views
|
631
|
-
result = execute <<~SQL
|
632
|
-
SELECT
|
633
|
-
schemaname || '.' || matviewname AS table_name
|
634
|
-
FROM
|
635
|
-
pg_matviews
|
636
|
-
SQL
|
637
|
-
result.map { |r| r["table_name"] }
|
638
|
-
end
|
639
|
-
|
640
|
-
def server_version_num
|
641
|
-
execute("SHOW server_version_num").first["server_version_num"].to_i
|
642
|
-
end
|
643
|
-
|
644
|
-
def database_view_tables
|
645
|
-
result = execute <<~SQL
|
646
|
-
SELECT
|
647
|
-
schemaname || '.' || viewname AS table_name,
|
648
|
-
definition
|
649
|
-
FROM
|
650
|
-
pg_views
|
651
|
-
WHERE
|
652
|
-
schemaname NOT IN ('information_schema', 'pg_catalog')
|
653
|
-
SQL
|
654
|
-
|
655
|
-
view_tables = {}
|
656
|
-
result.each do |row|
|
657
|
-
begin
|
658
|
-
view_tables[row["table_name"]] = PgQuery.parse(row["definition"]).tables
|
659
|
-
rescue PgQuery::ParseError
|
660
|
-
if @log_level.start_with?("debug")
|
661
|
-
log colorize("ERROR: Cannot parse view definition: #{row["table_name"]}", :red)
|
662
|
-
end
|
663
|
-
end
|
664
|
-
end
|
665
|
-
|
666
|
-
view_tables
|
667
|
-
end
|
668
|
-
|
669
|
-
def stat_statements
|
670
|
-
total_time = server_version_num >= 130000 ? "(total_plan_time + total_exec_time)" : "total_time"
|
671
|
-
sql = <<~SQL
|
672
|
-
SELECT
|
673
|
-
DISTINCT query
|
674
|
-
FROM
|
675
|
-
pg_stat_statements
|
676
|
-
INNER JOIN
|
677
|
-
pg_database ON pg_database.oid = pg_stat_statements.dbid
|
678
|
-
WHERE
|
679
|
-
datname = current_database()
|
680
|
-
AND #{total_time} >= \$1
|
681
|
-
AND calls >= \$2
|
682
|
-
ORDER BY
|
683
|
-
1
|
684
|
-
SQL
|
685
|
-
execute(sql, params: [@min_time * 60000, @min_calls.to_i]).map { |q| q["query"] }
|
686
|
-
end
|
687
|
-
|
688
|
-
def with_advisory_lock
|
689
|
-
lock_id = 123456
|
690
|
-
first_time = true
|
691
|
-
while execute("SELECT pg_try_advisory_lock($1)", params: [lock_id]).first["pg_try_advisory_lock"] != "t"
|
692
|
-
if first_time
|
693
|
-
log "Waiting for lock..."
|
694
|
-
first_time = false
|
695
|
-
end
|
696
|
-
sleep(1)
|
697
|
-
end
|
698
|
-
yield
|
699
|
-
ensure
|
700
|
-
suppress_messages do
|
701
|
-
execute("SELECT pg_advisory_unlock($1)", params: [lock_id])
|
702
|
-
end
|
703
|
-
end
|
704
|
-
|
705
|
-
def suppress_messages
|
706
|
-
conn.set_notice_processor do |message|
|
707
|
-
# do nothing
|
708
|
-
end
|
709
|
-
yield
|
710
|
-
ensure
|
711
|
-
# clear notice processor
|
712
|
-
conn.set_notice_processor
|
713
|
-
end
|
714
|
-
|
715
|
-
def index_exists?(index)
|
716
|
-
indexes([index[:table]]).find { |i| i["columns"] == index[:columns] }
|
717
|
-
end
|
718
|
-
|
719
|
-
def columns(tables)
|
720
|
-
query = <<~SQL
|
721
|
-
SELECT
|
722
|
-
s.nspname || '.' || t.relname AS table_name,
|
723
|
-
a.attname AS column_name,
|
724
|
-
pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type
|
725
|
-
FROM pg_attribute a
|
726
|
-
JOIN pg_class t on a.attrelid = t.oid
|
727
|
-
JOIN pg_namespace s on t.relnamespace = s.oid
|
728
|
-
WHERE a.attnum > 0
|
729
|
-
AND NOT a.attisdropped
|
730
|
-
AND s.nspname || '.' || t.relname IN (#{tables.size.times.map { |i| "$#{i + 1}" }.join(", ")})
|
731
|
-
ORDER BY
|
732
|
-
1, 2
|
733
|
-
SQL
|
734
|
-
columns = execute(query, params: tables.to_a)
|
735
|
-
columns.map { |v| {table: v["table_name"], column: v["column_name"], type: v["data_type"]} }
|
499
|
+
def create_hypothetical_index(table, columns)
|
500
|
+
execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{@connection.quote_ident(table)} (#{columns.map { |c| @connection.quote_ident(c) }.join(", ")})')").first["indexname"]
|
736
501
|
end
|
737
502
|
|
738
503
|
def indexes(tables)
|
@@ -761,10 +526,6 @@ module Dexter
|
|
761
526
|
execute(query, params: tables.to_a).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
|
762
527
|
end
|
763
528
|
|
764
|
-
def search_path
|
765
|
-
execute("SELECT current_schemas(true)")[0]["current_schemas"][1..-2].split(",")
|
766
|
-
end
|
767
|
-
|
768
529
|
def unquote(part)
|
769
530
|
if part && part.start_with?('"') && part.end_with?('"')
|
770
531
|
part[1..-2]
|
@@ -773,15 +534,6 @@ module Dexter
|
|
773
534
|
end
|
774
535
|
end
|
775
536
|
|
776
|
-
def quote_ident(value)
|
777
|
-
value.split(".").map { |v| conn.quote_ident(v) }.join(".")
|
778
|
-
end
|
779
|
-
|
780
|
-
# from activesupport
|
781
|
-
def squish(str)
|
782
|
-
str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
|
783
|
-
end
|
784
|
-
|
785
537
|
def safe_statement(statement)
|
786
538
|
statement.gsub(";", "")
|
787
539
|
end
|