pgdexter 0.5.6 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,77 +2,77 @@ module Dexter
2
2
  class Indexer
3
3
  include Logging
4
4
 
5
- def initialize(options)
5
+ def initialize(connection:, **options)
6
+ @connection = connection
6
7
  @create = options[:create]
7
8
  @tablespace = options[:tablespace]
8
9
  @log_level = options[:log_level]
9
10
  @exclude_tables = options[:exclude]
10
11
  @include_tables = Array(options[:include].split(",")) if options[:include]
11
- @log_sql = options[:log_sql]
12
12
  @log_explain = options[:log_explain]
13
- @min_time = options[:min_time] || 0
14
- @min_calls = options[:min_calls] || 0
15
13
  @analyze = options[:analyze]
14
+ @min_cost = options[:min_cost].to_i
16
15
  @min_cost_savings_pct = options[:min_cost_savings_pct].to_i
17
16
  @options = options
18
- @mutex = Mutex.new
19
-
20
- if server_version_num < 110000
21
- raise Dexter::Abort, "This version of Dexter requires Postgres 11+"
22
- end
23
-
24
- check_extension
25
-
26
- execute("SET lock_timeout = '5s'")
27
- end
28
-
29
- def process_stat_statements
30
- queries = stat_statements.map { |q| Query.new(q) }.sort_by(&:fingerprint).group_by(&:fingerprint).map { |_, v| v.first }
31
- log "Processing #{queries.size} new query fingerprints"
32
- process_queries(queries)
17
+ @server_version_num = @connection.server_version_num
33
18
  end
34
19
 
20
+ # TODO recheck server version?
35
21
  def process_queries(queries)
36
- # reset hypothetical indexes
37
- reset_hypothetical_indexes
38
-
39
- tables = Set.new(database_tables + materialized_views)
22
+ TableResolver.new(@connection, queries, log_level: @log_level).perform
23
+ candidate_queries = queries.reject(&:missing_tables)
40
24
 
41
- # map tables without schema to schema
42
- no_schema_tables = {}
43
- search_path_index = Hash[search_path.map.with_index.to_a]
44
- tables.group_by { |t| t.split(".")[-1] }.each do |group, t2|
45
- no_schema_tables[group] = t2.sort_by { |t| [search_path_index[t.split(".")[0]] || 1000000, t] }[0]
25
+ tables = determine_tables(candidate_queries)
26
+ candidate_queries.each do |query|
27
+ query.candidate_tables = query.tables.select { |t| tables.include?(t) }.sort
46
28
  end
29
+ candidate_queries.select! { |q| q.candidate_tables.any? }
47
30
 
48
- # add tables from views
49
- view_tables = database_view_tables
50
- view_tables.each do |v, vt|
51
- view_tables[v] = vt.map { |t| no_schema_tables[t] || t }
52
- end
31
+ if tables.any?
32
+ # analyze tables if needed
33
+ # TODO remove @log_level in 0.7.0
34
+ analyze_tables(tables) if @analyze || @log_level == "debug2"
35
+
36
+ # get initial costs for queries
37
+ reset_hypothetical_indexes
38
+ calculate_plan(candidate_queries)
39
+ candidate_queries.select! { |q| q.initial_cost && q.initial_cost >= @min_cost }
40
+
41
+ # find columns
42
+ ColumnResolver.new(@connection, candidate_queries, log_level: @log_level).perform
43
+ candidate_queries.each do |query|
44
+ # no reason to use btree index for json columns
45
+ # TODO check type supports btree
46
+ query.candidate_columns = query.columns.reject { |c| ["json", "jsonb", "point"].include?(c[:type]) }.sort_by { |c| [c[:table], c[:column]] }
47
+ end
48
+ candidate_queries.select! { |q| q.candidate_columns.any? }
53
49
 
54
- # fully resolve tables
55
- # make sure no views in result
56
- view_tables.each do |v, vt|
57
- view_tables[v] = vt.flat_map { |t| view_tables[t] || [t] }.uniq
50
+ # create hypothetical indexes and explain queries
51
+ batch_hypothetical_indexes(candidate_queries)
58
52
  end
59
53
 
60
- # filter queries from other databases and system tables
61
- queries.each do |query|
62
- # add schema to table if needed
63
- query.tables = query.tables.map { |t| no_schema_tables[t] || t }
54
+ # see if new indexes were used and meet bar
55
+ new_indexes = determine_indexes(queries, tables)
64
56
 
65
- # substitute view tables
66
- new_tables = query.tables.flat_map { |t| view_tables[t] || [t] }.uniq
67
- query.tables_from_views = new_tables - query.tables
68
- query.tables = new_tables
57
+ # display new indexes
58
+ show_new_indexes(new_indexes)
69
59
 
70
- # check for missing tables
71
- query.missing_tables = !query.tables.all? { |t| tables.include?(t) }
72
- end
60
+ # display debug info
61
+ show_debug_info(new_indexes, queries) if @log_level.start_with?("debug")
62
+
63
+ # create new indexes
64
+ IndexCreator.new(@connection, self, new_indexes, @tablespace).perform if @create && new_indexes.any?
65
+ end
66
+
67
+ private
73
68
 
69
+ def reset_hypothetical_indexes
70
+ execute("SELECT hypopg_reset()")
71
+ end
72
+
73
+ def determine_tables(candidate_queries)
74
74
  # set tables
75
- tables = Set.new(queries.reject(&:missing_tables).flat_map(&:tables))
75
+ tables = Set.new(candidate_queries.flat_map(&:tables))
76
76
 
77
77
  # must come after missing tables set
78
78
  if @include_tables
@@ -88,54 +88,10 @@ module Dexter
88
88
  # remove system tables
89
89
  tables.delete_if { |t| t.start_with?("information_schema.", "pg_catalog.") }
90
90
 
91
- queries.each do |query|
92
- query.candidate_tables = !query.missing_tables && query.tables.any? { |t| tables.include?(t) }
93
- end
94
-
95
- # analyze tables if needed
96
- analyze_tables(tables) if tables.any? && (@analyze || @log_level == "debug2")
97
-
98
- # create hypothetical indexes and explain queries
99
- if tables.any?
100
- # process in batches to prevent "hypopg: not more oid available" error
101
- # https://hypopg.readthedocs.io/en/rel1_stable/usage.html#configuration
102
- queries.select(&:candidate_tables).each_slice(500) do |batch|
103
- create_hypothetical_indexes(batch)
104
- end
105
- end
106
-
107
- # see if new indexes were used and meet bar
108
- new_indexes = determine_indexes(queries, tables)
109
-
110
- # display and create new indexes
111
- show_and_create_indexes(new_indexes, queries)
112
- end
113
-
114
- private
115
-
116
- def check_extension
117
- extension = execute("SELECT installed_version FROM pg_available_extensions WHERE name = 'hypopg'").first
118
-
119
- if extension.nil?
120
- raise Dexter::Abort, "Install HypoPG first: https://github.com/ankane/dexter#installation"
121
- end
122
-
123
- if extension["installed_version"].nil?
124
- if @options[:enable_hypopg]
125
- execute("CREATE EXTENSION hypopg")
126
- else
127
- raise Dexter::Abort, "Run `CREATE EXTENSION hypopg` or pass --enable-hypopg"
128
- end
129
- end
91
+ tables
130
92
  end
131
93
 
132
- def reset_hypothetical_indexes
133
- execute("SELECT hypopg_reset()")
134
- end
135
-
136
- def analyze_tables(tables)
137
- tables = tables.to_a.sort
138
-
94
+ def analyze_stats(tables)
139
95
  query = <<~SQL
140
96
  SELECT
141
97
  schemaname || '.' || relname AS table,
@@ -146,10 +102,14 @@ module Dexter
146
102
  WHERE
147
103
  schemaname || '.' || relname IN (#{tables.size.times.map { |i| "$#{i + 1}" }.join(", ")})
148
104
  SQL
149
- analyze_stats = execute(query, params: tables.to_a)
105
+ execute(query, params: tables)
106
+ end
107
+
108
+ def analyze_tables(tables)
109
+ tables = tables.to_a.sort
150
110
 
151
111
  last_analyzed = {}
152
- analyze_stats.each do |stats|
112
+ analyze_stats(tables).each do |stats|
153
113
  last_analyzed[stats["table"]] = Time.parse(stats["last_analyze"]) if stats["last_analyze"]
154
114
  end
155
115
 
@@ -162,7 +122,7 @@ module Dexter
162
122
  end
163
123
 
164
124
  if @analyze && (!la || la < Time.now - 3600)
165
- statement = "ANALYZE #{quote_ident(table)}"
125
+ statement = "ANALYZE #{@connection.quote_ident(table)}"
166
126
  log "Running analyze: #{statement}"
167
127
  execute(statement)
168
128
  end
@@ -186,71 +146,83 @@ module Dexter
186
146
  end
187
147
  end
188
148
 
189
- def create_hypothetical_indexes(queries)
190
- candidates = {}
149
+ # process in batches to prevent "hypopg: not more oid available" error
150
+ # https://hypopg.readthedocs.io/en/rel1_stable/usage.html#configuration
151
+ def batch_hypothetical_indexes(candidate_queries)
152
+ batch_count = 0
153
+ batch = []
154
+ single_column_indexes = Set.new
155
+ multicolumn_indexes = Set.new
191
156
 
192
- reset_hypothetical_indexes
157
+ # sort to improve batching
158
+ # TODO improve
159
+ candidate_queries.sort_by! { |q| q.candidate_columns.map { |c| [c[:table], c[:column]] } }
193
160
 
194
- # get initial costs for queries
195
- calculate_plan(queries)
196
- explainable_queries = queries.select { |q| q.plans.any? && q.high_cost? }
161
+ candidate_queries.each do |query|
162
+ batch << query
197
163
 
198
- # filter tables for performance
199
- tables = Set.new(explainable_queries.flat_map(&:tables))
200
- tables_from_views = Set.new(explainable_queries.flat_map(&:tables_from_views))
164
+ single_column_indexes.merge(query.candidate_columns)
201
165
 
202
- if tables.any?
203
- # since every set of multi-column indexes are expensive
204
- # try to parse out columns
205
- possible_columns = Set.new
206
- explainable_queries.each do |query|
207
- log "Finding columns: #{query.statement}" if @log_level == "debug3"
208
- begin
209
- find_columns(query.tree).each do |col|
210
- last_col = col["fields"].last
211
- if last_col["String"]
212
- possible_columns << last_col["String"]["sval"]
213
- end
214
- end
215
- rescue JSON::NestingError
216
- if @log_level.start_with?("debug")
217
- log colorize("ERROR: Cannot get columns", :red)
218
- end
219
- end
166
+ # TODO for multicolumn indexes, use ordering
167
+ columns_by_table = query.candidate_columns.group_by { |c| c[:table] }
168
+ columns_by_table.each do |_, columns|
169
+ multicolumn_indexes.merge(columns.permutation(2).to_a)
220
170
  end
221
171
 
222
- # create hypothetical indexes
223
- # use all columns in tables from views
224
- columns_by_table = columns(tables).select { |c| possible_columns.include?(c[:column]) || tables_from_views.include?(c[:table]) }.group_by { |c| c[:table] }
172
+ if single_column_indexes.size + multicolumn_indexes.size >= 500
173
+ create_hypothetical_indexes(batch, single_column_indexes, multicolumn_indexes, batch_count)
174
+ batch_count += 1
175
+ batch.clear
176
+ single_column_indexes.clear
177
+ multicolumn_indexes.clear
178
+ end
179
+ end
225
180
 
226
- # create single column indexes
227
- create_hypothetical_indexes_helper(columns_by_table, 1, candidates)
181
+ if batch.any?
182
+ create_hypothetical_indexes(batch, single_column_indexes, multicolumn_indexes, batch_count)
183
+ end
184
+ end
228
185
 
229
- # get next round of costs
230
- calculate_plan(explainable_queries)
186
+ def create_candidate_indexes(candidate_indexes, index_mapping)
187
+ candidate_indexes.each do |columns|
188
+ begin
189
+ index_name = create_hypothetical_index(columns[0][:table], columns.map { |c| c[:column] })
190
+ index_mapping[index_name] = columns
191
+ rescue PG::UndefinedObject
192
+ # data type x has no default operator class for access method "btree"
193
+ end
194
+ end
195
+ rescue PG::InternalError
196
+ # hypopg: not more oid available
197
+ log colorize("WARNING: Limiting index candidates", :yellow) if @log_level == "debug2"
198
+ end
231
199
 
232
- # create multicolumn indexes
233
- create_hypothetical_indexes_helper(columns_by_table, 2, candidates)
200
+ def create_hypothetical_indexes(queries, single_column_indexes, multicolumn_indexes, batch_count)
201
+ index_mapping = {}
202
+ reset_hypothetical_indexes
234
203
 
235
- # get next round of costs
236
- calculate_plan(explainable_queries)
237
- end
204
+ # check single column indexes
205
+ create_candidate_indexes(single_column_indexes.map { |c| [c] }, index_mapping)
206
+ calculate_plan(queries)
207
+
208
+ # check multicolumn indexes
209
+ create_candidate_indexes(multicolumn_indexes, index_mapping)
210
+ calculate_plan(queries)
238
211
 
212
+ # save index mapping for analysis
239
213
  queries.each do |query|
240
- query.candidates = candidates
214
+ query.index_mapping = index_mapping
241
215
  end
242
- end
243
216
 
244
- def find_columns(plan)
245
- plan = JSON.parse(plan.to_json, max_nesting: 1000)
246
- find_by_key(plan, "ColumnRef")
217
+ # TODO different log level?
218
+ log "Batch #{batch_count + 1}: #{queries.size} queries, #{index_mapping.size} hypothetical indexes" if @log_level == "debug2"
247
219
  end
248
220
 
249
221
  def find_indexes(plan)
250
- find_by_key(plan, "Index Name")
222
+ self.class.find_by_key(plan, "Index Name")
251
223
  end
252
224
 
253
- def find_by_key(plan, key)
225
+ def self.find_by_key(plan, key)
254
226
  result = []
255
227
  queue = [plan]
256
228
  while queue.any?
@@ -271,16 +243,16 @@ module Dexter
271
243
  result
272
244
  end
273
245
 
274
- def hypo_indexes_from_plan(index_name_to_columns, plan, index_set)
246
+ def hypo_indexes_from_plan(index_mapping, plan, index_set)
275
247
  query_indexes = []
276
248
 
277
249
  find_indexes(plan).uniq.sort.each do |index_name|
278
- col_set = index_name_to_columns[index_name]
250
+ columns = index_mapping[index_name]
279
251
 
280
- if col_set
252
+ if columns
281
253
  index = {
282
- table: col_set[0][:table],
283
- columns: col_set.map { |c| c[:column] }
254
+ table: columns[0][:table],
255
+ columns: columns.map { |c| c[:column] }
284
256
  }
285
257
 
286
258
  unless index_set.include?([index[:table], index[:columns]])
@@ -310,10 +282,10 @@ module Dexter
310
282
  end
311
283
  end
312
284
 
313
- savings_ratio = (1 - @min_cost_savings_pct / 100.0)
285
+ savings_ratio = 1 - @min_cost_savings_pct / 100.0
314
286
 
315
287
  queries.each do |query|
316
- if query.explainable? && query.high_cost?
288
+ if query.fully_analyzed?
317
289
  new_cost, new_cost2 = query.costs[1..2]
318
290
 
319
291
  cost_savings = new_cost < query.initial_cost * savings_ratio
@@ -322,11 +294,11 @@ module Dexter
322
294
  cost_savings2 = new_cost > 100 && new_cost2 < new_cost * savings_ratio
323
295
 
324
296
  key = cost_savings2 ? 2 : 1
325
- query_indexes = hypo_indexes_from_plan(query.candidates, query.plans[key], index_set)
297
+ query_indexes = hypo_indexes_from_plan(query.index_mapping, query.plans[key], index_set)
326
298
 
327
299
  # likely a bad suggestion, so try single column
328
300
  if cost_savings2 && query_indexes.size > 1
329
- query_indexes = hypo_indexes_from_plan(query.candidates, query.plans[1], index_set)
301
+ query_indexes = hypo_indexes_from_plan(query.index_mapping, query.plans[1], index_set)
330
302
  cost_savings2 = false
331
303
  end
332
304
 
@@ -348,7 +320,7 @@ module Dexter
348
320
 
349
321
  query_indexes.each do |query_index|
350
322
  reset_hypothetical_indexes
351
- create_hypothetical_index(query_index[:table], query_index[:columns].map { |v| {column: v} })
323
+ create_hypothetical_index(query_index[:table], query_index[:columns])
352
324
  plan3 = plan(query.statement)
353
325
  cost3 = plan3["Total Cost"]
354
326
 
@@ -399,8 +371,8 @@ module Dexter
399
371
 
400
372
  # TODO optimize
401
373
  if @log_level.start_with?("debug")
402
- query.pass1_indexes = hypo_indexes_from_plan(query.candidates, query.plans[1], index_set)
403
- query.pass2_indexes = hypo_indexes_from_plan(query.candidates, query.plans[2], index_set)
374
+ query.pass1_indexes = hypo_indexes_from_plan(query.index_mapping, query.plans[1], index_set)
375
+ query.pass2_indexes = hypo_indexes_from_plan(query.index_mapping, query.plans[2], index_set)
404
376
  end
405
377
  end
406
378
  end
@@ -424,8 +396,7 @@ module Dexter
424
396
  end
425
397
  end
426
398
 
427
- def show_and_create_indexes(new_indexes, queries)
428
- # print summary
399
+ def show_new_indexes(new_indexes)
429
400
  if new_indexes.any?
430
401
  new_indexes.each do |index|
431
402
  log colorize("Index found: #{index[:table]} (#{index[:columns].join(", ")})", :green)
@@ -433,121 +404,56 @@ module Dexter
433
404
  else
434
405
  log "No new indexes found"
435
406
  end
407
+ end
436
408
 
437
- # debug info
438
- if @log_level.start_with?("debug")
439
- index_queries = new_indexes.flat_map { |i| i[:queries].sort_by(&:fingerprint) }
440
- if @log_level == "debug2"
441
- fingerprints = Set.new(index_queries.map(&:fingerprint))
442
- index_queries.concat(queries.reject { |q| fingerprints.include?(q.fingerprint) }.sort_by(&:fingerprint))
443
- end
444
- index_queries.each do |query|
445
- log "-" * 80
446
- log "Query #{query.fingerprint}"
447
- log "Total time: #{(query.total_time / 60000.0).round(1)} min, avg time: #{(query.total_time / query.calls.to_f).round} ms, calls: #{query.calls}" if query.total_time
448
-
449
- if query.fingerprint == "unknown"
450
- log "Could not parse query"
451
- elsif query.tables.empty?
452
- log "No tables"
453
- elsif query.missing_tables
454
- log "Tables not present in current database"
455
- elsif !query.candidate_tables
456
- log "No candidate tables for indexes"
457
- elsif query.explainable? && !query.high_cost?
458
- log "Low initial cost: #{query.initial_cost}"
459
- elsif query.explainable?
460
- query_indexes = query.indexes || []
461
- log "Start: #{query.costs[0]}"
462
- log "Pass1: #{query.costs[1]} : #{log_indexes(query.pass1_indexes || [])}"
463
- log "Pass2: #{query.costs[2]} : #{log_indexes(query.pass2_indexes || [])}"
464
- if query.costs[3]
465
- log "Pass3: #{query.costs[3]} : #{log_indexes(query.pass3_indexes || [])}"
466
- end
467
- log "Final: #{query.new_cost} : #{log_indexes(query.suggest_index ? query_indexes : [])}"
468
- if (query.pass1_indexes.any? || query.pass2_indexes.any?) && !query.suggest_index
469
- log "Need #{@min_cost_savings_pct}% cost savings to suggest index"
470
- end
471
- else
472
- log "Could not run explain"
409
+ def show_debug_info(new_indexes, queries)
410
+ index_queries = new_indexes.flat_map { |i| i[:queries].sort_by(&:fingerprint) }
411
+ if @log_level == "debug2"
412
+ fingerprints = Set.new(index_queries.map(&:fingerprint))
413
+ index_queries.concat(queries.reject { |q| fingerprints.include?(q.fingerprint) }.sort_by(&:fingerprint))
414
+ end
415
+ index_queries.each do |query|
416
+ log "-" * 80
417
+ log "Query #{query.fingerprint}"
418
+ log "Total time: #{(query.total_time / 60000.0).round(1)} min, avg time: #{(query.total_time / query.calls.to_f).round} ms, calls: #{query.calls}" if query.calls > 0
419
+
420
+ if query.fingerprint == "unknown"
421
+ log "Could not parse query"
422
+ elsif query.tables.empty?
423
+ log "No tables"
424
+ elsif query.missing_tables
425
+ log "Tables not present in current database"
426
+ elsif query.candidate_tables.empty?
427
+ log "No candidate tables for indexes"
428
+ elsif !query.initial_cost
429
+ log "Could not run explain"
430
+ elsif query.initial_cost < @min_cost
431
+ log "Low initial cost: #{query.initial_cost}"
432
+ elsif query.candidate_columns.empty?
433
+ log "No candidate columns for indexes"
434
+ elsif query.fully_analyzed?
435
+ query_indexes = query.indexes || []
436
+ log "Start: #{query.costs[0]}"
437
+ log "Pass1: #{query.costs[1]} : #{log_indexes(query.pass1_indexes || [])}"
438
+ log "Pass2: #{query.costs[2]} : #{log_indexes(query.pass2_indexes || [])}"
439
+ if query.costs[3]
440
+ log "Pass3: #{query.costs[3]} : #{log_indexes(query.pass3_indexes || [])}"
473
441
  end
474
- log
475
- log query.statement
476
- log
477
- end
478
- end
479
-
480
- # create
481
- if @create && new_indexes.any?
482
- # 1. create lock
483
- # 2. refresh existing index list
484
- # 3. create indexes that still don't exist
485
- # 4. release lock
486
- with_advisory_lock do
487
- new_indexes.each do |index|
488
- unless index_exists?(index)
489
- statement = String.new("CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})")
490
- statement << " TABLESPACE #{quote_ident(@tablespace)}" if @tablespace
491
- log "Creating index: #{statement}"
492
- started_at = monotonic_time
493
- begin
494
- execute(statement)
495
- log "Index created: #{((monotonic_time - started_at) * 1000).to_i} ms"
496
- rescue PG::LockNotAvailable
497
- log "Could not acquire lock: #{index[:table]}"
498
- end
499
- end
442
+ log "Final: #{query.new_cost} : #{log_indexes(query.suggest_index ? query_indexes : [])}"
443
+ if (query.pass1_indexes.any? || query.pass2_indexes.any?) && !query.suggest_index
444
+ log "Need #{@min_cost_savings_pct}% cost savings to suggest index"
500
445
  end
501
- end
502
- end
503
-
504
- new_indexes
505
- end
506
-
507
- def monotonic_time
508
- Process.clock_gettime(Process::CLOCK_MONOTONIC)
509
- end
510
-
511
- def conn
512
- @conn ||= begin
513
- # set connect timeout if none set
514
- ENV["PGCONNECT_TIMEOUT"] ||= "3"
515
-
516
- if @options[:dbname].start_with?("postgres://", "postgresql://")
517
- config = @options[:dbname]
518
446
  else
519
- config = {
520
- host: @options[:host],
521
- port: @options[:port],
522
- dbname: @options[:dbname],
523
- user: @options[:username]
524
- }.reject { |_, value| value.to_s.empty? }
525
- config = config[:dbname] if config.keys == [:dbname] && config[:dbname].include?("=")
447
+ log "Could not run explain"
526
448
  end
527
- PG::Connection.new(config)
449
+ log
450
+ log query.statement
451
+ log
528
452
  end
529
- rescue PG::ConnectionBad => e
530
- raise Dexter::Abort, e.message
531
453
  end
532
454
 
533
- def execute(query, pretty: true, params: [], use_exec: false)
534
- # use exec_params instead of exec when possible for security
535
- #
536
- # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
537
- # (There can be semicolons in it, but not more than one nonempty command.)
538
- # This is a limitation of the underlying protocol, but has some usefulness
539
- # as an extra defense against SQL-injection attacks.
540
- # https://www.postgresql.org/docs/current/static/libpq-exec.html
541
- query = squish(query) if pretty
542
- log colorize("[sql] #{query}#{params.any? ? " /*#{params.to_json}*/" : ""}", :cyan) if @log_sql
543
-
544
- @mutex.synchronize do
545
- if use_exec
546
- conn.exec("#{query} /*dexter*/").to_a
547
- else
548
- conn.exec_params("#{query} /*dexter*/", params).to_a
549
- end
550
- end
455
+ def execute(...)
456
+ @connection.execute(...)
551
457
  end
552
458
 
553
459
  def plan(query)
@@ -557,7 +463,7 @@ module Dexter
557
463
  # try to EXPLAIN normalized queries
558
464
  # https://dev.to/yugabyte/explain-from-pgstatstatements-normalized-queries-how-to-always-get-the-generic-plan-in--5cfi
559
465
  normalized = query.include?("$1")
560
- generic_plan = normalized && server_version_num >= 160000
466
+ generic_plan = normalized && @server_version_num >= 160000
561
467
  explain_normalized = normalized && !generic_plan
562
468
  if explain_normalized
563
469
  prepared_name = "dexter_prepared"
@@ -569,16 +475,7 @@ module Dexter
569
475
  execute("BEGIN")
570
476
  transaction = true
571
477
 
572
- if server_version_num >= 120000
573
- execute("SET LOCAL plan_cache_mode = force_generic_plan")
574
- else
575
- execute("SET LOCAL cpu_operator_cost = 1e42")
576
- 5.times do
577
- execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}", pretty: false)
578
- end
579
- execute("ROLLBACK")
580
- execute("BEGIN")
581
- end
478
+ execute("SET LOCAL plan_cache_mode = force_generic_plan")
582
479
  end
583
480
 
584
481
  explain_prefix = generic_plan ? "GENERIC_PLAN, " : ""
@@ -599,140 +496,8 @@ module Dexter
599
496
  end
600
497
  end
601
498
 
602
- # TODO for multicolumn indexes, use ordering
603
- def create_hypothetical_indexes_helper(columns_by_table, n, candidates)
604
- columns_by_table.each do |table, cols|
605
- # no reason to use btree index for json columns
606
- cols.reject { |c| ["json", "jsonb"].include?(c[:type]) }.permutation(n) do |col_set|
607
- index_name = create_hypothetical_index(table, col_set)
608
- candidates[index_name] = col_set
609
- end
610
- end
611
- end
612
-
613
- def create_hypothetical_index(table, col_set)
614
- execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(table)} (#{col_set.map { |c| quote_ident(c[:column]) }.join(", ")})')").first["indexname"]
615
- end
616
-
617
- def database_tables
618
- result = execute <<~SQL
619
- SELECT
620
- table_schema || '.' || table_name AS table_name
621
- FROM
622
- information_schema.tables
623
- WHERE
624
- table_catalog = current_database()
625
- AND table_type IN ('BASE TABLE', 'VIEW')
626
- SQL
627
- result.map { |r| r["table_name"] }
628
- end
629
-
630
- def materialized_views
631
- result = execute <<~SQL
632
- SELECT
633
- schemaname || '.' || matviewname AS table_name
634
- FROM
635
- pg_matviews
636
- SQL
637
- result.map { |r| r["table_name"] }
638
- end
639
-
640
- def server_version_num
641
- execute("SHOW server_version_num").first["server_version_num"].to_i
642
- end
643
-
644
- def database_view_tables
645
- result = execute <<~SQL
646
- SELECT
647
- schemaname || '.' || viewname AS table_name,
648
- definition
649
- FROM
650
- pg_views
651
- WHERE
652
- schemaname NOT IN ('information_schema', 'pg_catalog')
653
- SQL
654
-
655
- view_tables = {}
656
- result.each do |row|
657
- begin
658
- view_tables[row["table_name"]] = PgQuery.parse(row["definition"]).tables
659
- rescue PgQuery::ParseError
660
- if @log_level.start_with?("debug")
661
- log colorize("ERROR: Cannot parse view definition: #{row["table_name"]}", :red)
662
- end
663
- end
664
- end
665
-
666
- view_tables
667
- end
668
-
669
- def stat_statements
670
- total_time = server_version_num >= 130000 ? "(total_plan_time + total_exec_time)" : "total_time"
671
- sql = <<~SQL
672
- SELECT
673
- DISTINCT query
674
- FROM
675
- pg_stat_statements
676
- INNER JOIN
677
- pg_database ON pg_database.oid = pg_stat_statements.dbid
678
- WHERE
679
- datname = current_database()
680
- AND #{total_time} >= \$1
681
- AND calls >= \$2
682
- ORDER BY
683
- 1
684
- SQL
685
- execute(sql, params: [@min_time * 60000, @min_calls.to_i]).map { |q| q["query"] }
686
- end
687
-
688
- def with_advisory_lock
689
- lock_id = 123456
690
- first_time = true
691
- while execute("SELECT pg_try_advisory_lock($1)", params: [lock_id]).first["pg_try_advisory_lock"] != "t"
692
- if first_time
693
- log "Waiting for lock..."
694
- first_time = false
695
- end
696
- sleep(1)
697
- end
698
- yield
699
- ensure
700
- suppress_messages do
701
- execute("SELECT pg_advisory_unlock($1)", params: [lock_id])
702
- end
703
- end
704
-
705
- def suppress_messages
706
- conn.set_notice_processor do |message|
707
- # do nothing
708
- end
709
- yield
710
- ensure
711
- # clear notice processor
712
- conn.set_notice_processor
713
- end
714
-
715
- def index_exists?(index)
716
- indexes([index[:table]]).find { |i| i["columns"] == index[:columns] }
717
- end
718
-
719
- def columns(tables)
720
- query = <<~SQL
721
- SELECT
722
- s.nspname || '.' || t.relname AS table_name,
723
- a.attname AS column_name,
724
- pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type
725
- FROM pg_attribute a
726
- JOIN pg_class t on a.attrelid = t.oid
727
- JOIN pg_namespace s on t.relnamespace = s.oid
728
- WHERE a.attnum > 0
729
- AND NOT a.attisdropped
730
- AND s.nspname || '.' || t.relname IN (#{tables.size.times.map { |i| "$#{i + 1}" }.join(", ")})
731
- ORDER BY
732
- 1, 2
733
- SQL
734
- columns = execute(query, params: tables.to_a)
735
- columns.map { |v| {table: v["table_name"], column: v["column_name"], type: v["data_type"]} }
499
+ def create_hypothetical_index(table, columns)
500
+ execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{@connection.quote_ident(table)} (#{columns.map { |c| @connection.quote_ident(c) }.join(", ")})')").first["indexname"]
736
501
  end
737
502
 
738
503
  def indexes(tables)
@@ -761,10 +526,6 @@ module Dexter
761
526
  execute(query, params: tables.to_a).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
762
527
  end
763
528
 
764
- def search_path
765
- execute("SELECT current_schemas(true)")[0]["current_schemas"][1..-2].split(",")
766
- end
767
-
768
529
  def unquote(part)
769
530
  if part && part.start_with?('"') && part.end_with?('"')
770
531
  part[1..-2]
@@ -773,15 +534,6 @@ module Dexter
773
534
  end
774
535
  end
775
536
 
776
- def quote_ident(value)
777
- value.split(".").map { |v| conn.quote_ident(v) }.join(".")
778
- end
779
-
780
- # from activesupport
781
- def squish(str)
782
- str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
783
- end
784
-
785
537
  def safe_statement(statement)
786
538
  statement.gsub(";", "")
787
539
  end