pgdexter 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78c0747c456e498798accd4e4881a8041d246e07
-  data.tar.gz: c4c9cffe3e04d07675c36abf0532db3634e356ea
+  metadata.gz: 714767248afe28ad9e354ebb6485b34a8d6aadd0
+  data.tar.gz: 545eaaea1df0312049f4cecff3fe9b03b80a6cd9
 SHA512:
-  metadata.gz: df826f95f34809bd94cb7450cc43d4bd6beed81606e6e7079842dd4a05e983d4041bf956645c09a290cf8fa37140917da1a12fe332d6268969da5310f688e148
-  data.tar.gz: f7015b408e7864bfe7e4e136cbb60acbe837cbbe640ec4f43dfb2d9631f2d4cb2b4c86826cb58051d68941bdfa6fd6625029babe7c1609e084db1a4f92a23652
+  metadata.gz: '096684887c5d5a48a2df74a2dae6cf70bc2b5e98dd5dc6d7d6d8b583bfb76193ce86166972d9553f89452f16629a8576a30f2627fd8d0b16a9672ce76a316835'
+  data.tar.gz: d128734845414672a3f470f138c6a422dc2b24024a66866e50524fe3bb3e949bd4a899305a93f71fb14af3b56820f66e20a00d5ba5788cd0c88bd0afaa064e32

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.2
+
+- Fixed parsing issue with named prepared statements
+- Fixed parsing issue with multiline queries in csv format
+- Better explanations for indexing decisions
+
 ## 0.3.1
 
 - Added support for queries with bind variables

data/exe/dexter

@@ -3,6 +3,8 @@
 require "dexter"
 begin
   Dexter::Client.new(ARGV).perform
+rescue Dexter::Abort => e
+  abort e.message
 rescue Interrupt => e
   # do nothing
 end

data/lib/dexter/client.rb

@@ -1,5 +1,7 @@
 module Dexter
   class Client
+    include Logging
+
     attr_reader :arguments, :options
 
     def initialize(args)
@@ -14,6 +16,7 @@ module Dexter
         query = Query.new(options[:statement])
         Indexer.new(options).process_queries([query])
       elsif options[:pg_stat_statements]
+        # TODO support streaming option
         Indexer.new(options).process_stat_statements
       elsif arguments.any?
         ARGV.replace(arguments)
@@ -35,12 +38,12 @@ Options:)
         o.string "--include", "only include specific tables"
         o.string "--input-format", "input format", default: "stderr"
         o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
-        o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
-        o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
-        o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
         o.boolean "--log-explain", "log explain", default: false, help: false
         o.string "--log-level", "log level", default: "info"
         o.boolean "--log-sql", "log sql", default: false
+        o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
+        o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
+        o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
         o.string "-s", "--statement", "process a single statement"
         # separator must go here to show up correctly - slop bug?
         o.separator ""
@@ -72,9 +75,5 @@ Options:)
     rescue Slop::Error => e
       abort e.message
     end
-
-    def log(message)
-      $stderr.puts message
-    end
   end
 end

data/lib/dexter/csv_log_parser.rb

@@ -2,10 +2,15 @@ require "csv"
 
 module Dexter
   class CsvLogParser < LogParser
+    FIRST_LINE_REGEX = /\A.+/
+
     def perform
-      CSV.new(@logfile).each do |row|
+      CSV.new(@logfile.to_io).each do |row|
         if (m = REGEX.match(row[13]))
-          active_line = m[3]
+          # replace first line with match
+          # needed for multiline queries
+          active_line = row[13].sub(FIRST_LINE_REGEX, m[3])
+
           add_parameters(active_line, row[14]) if row[14]
           process_entry(active_line, m[1].to_f)
         end

data/lib/dexter/indexer.rb

@@ -30,21 +30,11 @@ module Dexter
 
       tables = Set.new(database_tables)
 
-      if @include_tables
-        include_set = Set.new(@include_tables)
-        tables.keep_if { |t| include_set.include?(t) || include_set.include?(t.split(".")[-1]) }
-      end
-
-      if @exclude_tables.any?
-        exclude_set = Set.new(@exclude_tables)
-        tables.delete_if { |t| exclude_set.include?(t) || exclude_set.include?(t.split(".")[-1]) }
-      end
-
       # map tables without schema to schema
       no_schema_tables = {}
       search_path_index = Hash[search_path.map.with_index.to_a]
       tables.group_by { |t| t.split(".")[-1] }.each do |group, t2|
-        no_schema_tables[group] = t2.sort_by { |t| search_path_index[t.split(".")[0]] || 1000000 }[0]
+        no_schema_tables[group] = t2.sort_by { |t| [search_path_index[t.split(".")[0]] || 1000000, t] }[0]
       end
 
       # filter queries from other databases and system tables
@@ -59,11 +49,29 @@ module Dexter
       # set tables
       tables = Set.new(queries.reject(&:missing_tables).flat_map(&:tables))
 
+      # must come after missing tables set
+      if @include_tables
+        include_set = Set.new(@include_tables)
+        tables.keep_if { |t| include_set.include?(t) || include_set.include?(t.split(".")[-1]) }
+      end
+
+      if @exclude_tables.any?
+        exclude_set = Set.new(@exclude_tables)
+        tables.delete_if { |t| exclude_set.include?(t) || exclude_set.include?(t.split(".")[-1]) }
+      end
+
+      # remove system tables
+      tables.delete_if { |t| t.start_with?("information_schema.") || t.start_with?("pg_catalog.") }
+
+      queries.each do |query|
+        query.candidate_tables = !query.missing_tables && query.tables.any? { |t| tables.include?(t) }
+      end
+
       # analyze tables if needed
       analyze_tables(tables) if tables.any? && (@analyze || @log_level == "debug2")
 
       # create hypothetical indexes and explain queries
-      candidates = tables.any? ? create_hypothetical_indexes(queries.reject(&:missing_tables), tables) : {}
+      candidates = tables.any? ? create_hypothetical_indexes(queries.select(&:candidate_tables), tables) : {}
 
       # see if new indexes were used and meet bar
       new_indexes = determine_indexes(queries, candidates, tables)
@@ -344,7 +352,14 @@ module Dexter
           log "-" * 80
           log "Query #{query.fingerprint}"
           log "Total time: #{(query.total_time / 60000.0).round(1)} min, avg time: #{(query.total_time / query.calls.to_f).round} ms, calls: #{query.calls}" if query.total_time
-          if tables.empty?
+
+          if query.fingerprint == "unknown"
+            log "Could not parse query"
+          elsif query.tables.empty?
+            log "No tables"
+          elsif query.missing_tables
+            log "Tables not present in current database"
+          elsif !query.candidate_tables
             log "No candidate tables for indexes"
           elsif query.explainable? && !query.high_cost?
             log "Low initial cost: #{query.initial_cost}"
@@ -354,15 +369,9 @@ module Dexter
             log "Pass1: #{query.costs[1]} : #{log_indexes(query.pass1_indexes || [])}"
             log "Pass2: #{query.costs[2]} : #{log_indexes(query.pass2_indexes || [])}"
             log "Final: #{query.new_cost} : #{log_indexes(query.suggest_index ? query_indexes : [])}"
-            if query_indexes.any? && !query.suggest_index
+            if query_indexes.size == 1 && !query.suggest_index
               log "Need 50% cost savings to suggest index"
             end
-          elsif query.fingerprint == "unknown"
-            log "Could not parse query"
-          elsif query.tables.empty?
-            log "No tables"
-          elsif query.missing_tables
-            log "Tables not present in current database"
           else
             log "Could not run explain"
           end
@@ -452,9 +461,7 @@ module Dexter
         FROM
           information_schema.tables
         WHERE
-          table_catalog = current_database() AND
-          table_schema NOT IN ('pg_catalog', 'information_schema')
-          AND table_type = 'BASE TABLE'
+          table_catalog = current_database()
       SQL
       result.map { |r| r["table_name"] }
     end
@@ -549,7 +556,7 @@ module Dexter
     end
 
     def search_path
-      execute("SHOW search_path")[0]["search_path"].split(",").map(&:strip)
+      execute("SELECT current_schemas(true)")[0]["current_schemas"][1..-2].split(",")
     end
 
     def unquote(part)

data/lib/dexter/log_parser.rb

@@ -1,6 +1,8 @@
 module Dexter
   class LogParser
-    REGEX = /duration: (\d+\.\d+) ms  (statement|execute <unnamed>): (.+)/
+    include Logging
+
+    REGEX = /duration: (\d+\.\d+) ms  (statement|execute [^:]+): (.+)/
     LINE_SEPERATOR = ":  ".freeze
     DETAIL_LINE = "DETAIL:  ".freeze
 

data/lib/dexter/logging.rb

@@ -3,5 +3,9 @@ module Dexter
     def log(message = "")
       puts message unless $log_level == "error"
     end
+
+    def abort(message)
+      raise Dexter::Abort, message
+    end
   end
 end

data/lib/dexter/query.rb

@@ -2,7 +2,7 @@ module Dexter
   class Query
     attr_reader :statement, :fingerprint, :plans
     attr_writer :tables
-    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes
+    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :candidate_tables
 
     def initialize(statement, fingerprint = nil)
       @statement = statement

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/lib/dexter.rb

@@ -13,3 +13,7 @@ require "dexter/log_parser"
 require "dexter/csv_log_parser"
 require "dexter/processor"
 require "dexter/query"
+
+module Dexter
+  class Abort < StandardError; end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-29 00:00:00.000000000 Z
+date: 2018-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop