pgdexter 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c513808e2cdd9690c477c30548979fad122575a4
-  data.tar.gz: 6f4c605a0b68baa0275a523155b7be4ea9bc4d56
+  metadata.gz: 739c75ffdf977b9bbe8c29584da2c762f70fe527
+  data.tar.gz: 5059b53b96e5208146d3fffc44dcf485f40269e6
 SHA512:
-  metadata.gz: 3b38a53e96516a485394f3ec594ee0418c1a7ab94fe8726b3228cc5443843dd7f7ba2bd1fe4bb0cc5c4dbd877bd9b9b036d5bfd5b4cda275dc7176dbdf9da82c
-  data.tar.gz: cbe64d0cf9a40b96bf2644a80519bb837d235d6956ca9b61b24533867130958c95932ffbf4d69672218a10628c32b87a1f1e87f86a4968577f1629fc871ef440
+  metadata.gz: c9f071adbd8d2abe21dc454a709ddc2f0b7f9165473eac2a7de29ec4494e6cc17823d30fb000214d4d9f8d7c2cb327bef1a18e90418129868bf9c4011ad3b27c
+  data.tar.gz: e235bd08981cd3a0a2a75e266045bac412252bbab0a57623894f98e98442eabd0d8aa2923c8641dffbaed083aa1f3d026631f20983c411112502575052977676

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.1
+
+- Fixed bad suggestions
+- Improved debugging output
+
 ## 0.2.0
 
 - Added same connection options as `psql`

data/guides/Hosted-Postgres.md

@@ -8,12 +8,12 @@ Linux
 
 ```sh
 sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-sudo apt-get install wget ca-certificates
+sudo apt-get install -y wget ca-certificates
 wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
 sudo apt-get update
-sudo apt-get install postgresql-9.6 postgresql-server-dev-9.6
+sudo apt-get install -y postgresql-9.6 postgresql-server-dev-9.6
 sudo -u postgres createuser $(whoami) -s
-sudo apt-get install ruby2.2 ruby2.2-dev
+sudo apt-get install -y ruby2.2 ruby2.2-dev
 ```
 
 Mac
@@ -83,7 +83,7 @@ heroku logs -p postgres > postgresql.log
 We recommend creating a new instance from a snapshot for the dump to avoid affecting customers.
 
 ```sh
-pg_dump -v -j 8 -Fd -f /tmp/newout.dir <connection-string>
+pg_dump -v -j 8 -Fd -f /tmp/newout.dir <connection-options>
 ```
 
 Then shutdown the dump instance. Restore with:

data/lib/dexter/client.rb

@@ -39,6 +39,7 @@ Options:)
         o.string "--log-level", "log level", default: "info"
         o.boolean "--log-sql", "log sql", default: false
         o.string "-s", "--statement", "process a single statement"
+        # separator must go here to show up correctly - slop bug?
         o.separator ""
         o.separator "Connection options:"
         o.on "-v", "--version", "print the version" do

data/lib/dexter/collector.rb

@@ -37,7 +37,10 @@ module Dexter
       queries = []
       @top_queries.each do |k, v|
         if new_queries.include?(k) && v[:total_time] > @min_time
-          queries << Query.new(v[:query], k)
+          query = Query.new(v[:query], k)
+          query.total_time = v[:total_time]
+          query.calls = v[:calls]
+          queries << query
         end
       end
 

data/lib/dexter/indexer.rb

@@ -52,7 +52,7 @@ module Dexter
       new_indexes = determine_indexes(queries, candidates, tables)
 
       # display and create new indexes
-      show_and_create_indexes(new_indexes)
+      show_and_create_indexes(new_indexes, queries, tables)
     end
 
     private
@@ -133,13 +133,6 @@ module Dexter
       tables = Set.new(explainable_queries.flat_map(&:tables))
 
       if tables.any?
-        # get existing indexes
-        index_set = Set.new
-        indexes(tables).each do |index|
-          # TODO make sure btree
-          index_set << [index["table"], index["columns"]]
-        end
-
         # since every set of multi-column indexes are expensive
         # try to parse out columns
         possible_columns = Set.new
@@ -156,13 +149,13 @@ module Dexter
         columns_by_table = columns(tables).select { |c| possible_columns.include?(c[:column]) }.group_by { |c| c[:table] }
 
         # create single column indexes
-        create_hypothetical_indexes_helper(columns_by_table, 1, index_set, candidates)
+        create_hypothetical_indexes_helper(columns_by_table, 1, candidates)
 
         # get next round of costs
         calculate_plan(explainable_queries)
 
         # create multicolumn indexes
-        create_hypothetical_indexes_helper(columns_by_table, 2, index_set, candidates)
+        create_hypothetical_indexes_helper(columns_by_table, 2, candidates)
 
         # get next round of costs
         calculate_plan(explainable_queries)
@@ -196,88 +189,153 @@ module Dexter
       indexes
     end
 
+    def hypo_indexes_from_plan(index_name_to_columns, plan, index_set)
+      query_indexes = []
+
+      find_indexes(plan).uniq.sort.each do |index_name|
+        col_set = index_name_to_columns[index_name]
+
+        if col_set
+          index = {
+            table: col_set[0][:table],
+            columns: col_set.map { |c| c[:column] }
+          }
+
+          unless index_set.include?([index[:table], index[:columns]])
+            query_indexes << index
+          end
+        end
+      end
+
+      query_indexes
+    end
+
     def determine_indexes(queries, candidates, tables)
       new_indexes = {}
       index_name_to_columns = candidates.invert
 
+      # filter out existing indexes
+      # this must happen at end of process
+      # since sometimes hypothetical indexes
+      # can give lower cost than actual indexes
+      index_set = Set.new
+      if tables.any?
+        indexes(tables).each do |index|
+          if index["using"] == "btree"
+            # don't add indexes that are already covered
+            index_set << [index["table"], index["columns"].first(1)]
+            index_set << [index["table"], index["columns"].first(2)]
+          end
+        end
+      end
+
       queries.each do |query|
         if query.explainable? && query.high_cost?
           new_cost, new_cost2 = query.costs[1..2]
 
           cost_savings = new_cost < query.initial_cost * 0.5
+
           # set high bar for multicolumn indexes
           cost_savings2 = new_cost > 100 && new_cost2 < new_cost * 0.5
 
-          query.new_cost = cost_savings2 ? new_cost2 : new_cost
-
-          query_indexes = []
           key = cost_savings2 ? 2 : 1
-          indexes = find_indexes(query.plans[key]).uniq.sort
+          query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[key], index_set)
 
-          indexes.each do |index_name|
-            col_set = index_name_to_columns[index_name]
+          # likely a bad suggestion, so try single column
+          if cost_savings2 && query_indexes.size > 1
+            query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[1], index_set)
+            cost_savings2 = false
+          end
 
-            if col_set
-              index = {
-                table: col_set[0][:table],
-                columns: col_set.map { |c| c[:column] }
-              }
-              query_indexes << index
+          # TODO if multiple indexes are found (for either single or multicolumn)
+          # determine the impact of each individually
+          # for now, be conservative and don't suggest if more than one index
+          suggest_index = (cost_savings || cost_savings2) && query_indexes.size == 1
 
-              if cost_savings
-                new_indexes[index] ||= index.dup
-                (new_indexes[index][:queries] ||= []) << query
-              end
+          if suggest_index
+            query_indexes.each do |index|
+              new_indexes[index] ||= index.dup
+              (new_indexes[index][:queries] ||= []) << query
             end
           end
-        end
 
-        if @log_level == "debug2"
-          log "Processed #{query.fingerprint}"
-          if tables.empty?
-            log "No candidate tables for indexes"
-          elsif query.explainable? && !query.high_cost?
-            log "Low initial cost: #{query.initial_cost}"
-          elsif query.explainable?
-            log "Cost: #{query.initial_cost} -> #{query.new_cost}"
-
-            if query_indexes.any?
-              log "Indexes: #{query_indexes.map { |i| "#{i[:table]} (#{i[:columns].join(", ")})" }.join(", ")}"
-              log "Need 50% cost savings to suggest index" unless cost_savings || cost_savings2
+          query.indexes = query_indexes
+          query.suggest_index = suggest_index
+          query.new_cost =
+            if suggest_index
+              cost_savings2 ? new_cost2 : new_cost
             else
-              log "Indexes: None"
+              query.initial_cost
             end
-          elsif query.fingerprint == "unknown"
-            log "Could not parse query"
-          elsif query.tables.empty?
-            log "No tables"
-          elsif query.missing_tables
-            log "Tables not present in current database"
-          else
-            log "Could not run explain"
+
+          # TODO optimize
+          if @log_level.start_with?("debug")
+            query.pass1_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[1], index_set)
+            query.pass2_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[2], index_set)
           end
+        end
+      end
 
-          puts
-          puts query.statement
-          puts
+      # filter out covered indexes
+      covered = Set.new
+      new_indexes.values.each do |index|
+        if index[:columns].size > 1
+          covered << [index[:table], index[:columns].first(1)]
         end
       end
 
-      new_indexes.values.sort_by(&:to_a)
+      new_indexes.values.reject { |i| covered.include?([i[:table], i[:columns]]) }.sort_by(&:to_a)
     end
 
-    def show_and_create_indexes(new_indexes)
+    def log_indexes(indexes)
+      if indexes.any?
+        indexes.map { |i| "#{i[:table]} (#{i[:columns].join(", ")})" }.join(", ")
+      else
+        "None"
+      end
+    end
+
+    def show_and_create_indexes(new_indexes, queries, tables)
       if new_indexes.any?
         new_indexes.each do |index|
           log "Index found: #{index[:table]} (#{index[:columns].join(", ")})"
+        end
 
-          if @log_level.start_with?("debug")
-            index[:queries].sort_by(&:fingerprint).each do |query|
-              log "Query #{query.fingerprint} (Cost: #{query.initial_cost} -> #{query.new_cost})"
-              puts
-              puts query.statement
-              puts
+        if @log_level.start_with?("debug")
+          index_queries = new_indexes.flat_map { |i| i[:queries].sort_by(&:fingerprint) }
+          if @log_level == "debug2"
+            fingerprints = Set.new(index_queries.map(&:fingerprint))
+            index_queries.concat(queries.reject { |q| fingerprints.include?(q.fingerprint) }.sort_by(&:fingerprint))
+          end
+          index_queries.each do |query|
+            log "-" * 80
+            log "Query #{query.fingerprint}"
+            log "Total time: #{(query.total_time / 60000.0).round(1)} min, avg time: #{(query.total_time / query.calls.to_f).round} ms, calls: #{query.calls}" if query.total_time
+            if tables.empty?
+              log "No candidate tables for indexes"
+            elsif query.explainable? && !query.high_cost?
+              log "Low initial cost: #{query.initial_cost}"
+            elsif query.explainable?
+              query_indexes = query.indexes || []
+              log "Start: #{query.costs[0]}"
+              log "Pass1: #{query.costs[1]} : #{log_indexes(query.pass1_indexes || [])}"
+              log "Pass2: #{query.costs[2]} : #{log_indexes(query.pass2_indexes || [])}"
+              log "Final: #{query.new_cost} : #{log_indexes(query_indexes)}"
+              if query_indexes.any? && !query.suggest_index
+                log "Need 50% cost savings to suggest index"
+              end
+            elsif query.fingerprint == "unknown"
+              log "Could not parse query"
+            elsif query.tables.empty?
+              log "No tables"
+            elsif query.missing_tables
+              log "Tables not present in current database"
+            else
+              log "Could not run explain"
             end
+            log
+            log query.statement
+            log
           end
         end
 
@@ -347,13 +405,11 @@ module Dexter
     end
 
     # TODO for multicolumn indexes, use ordering
-    def create_hypothetical_indexes_helper(columns_by_table, n, index_set, candidates)
+    def create_hypothetical_indexes_helper(columns_by_table, n, candidates)
       columns_by_table.each do |table, cols|
         # no reason to use btree index for json columns
         cols.reject { |c| ["json", "jsonb"].include?(c[:type]) }.permutation(n) do |col_set|
-          if !index_set.include?([table, col_set.map { |col| col[:column] }])
-            candidates[col_set] = execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(table)} (#{col_set.map { |c| quote_ident(c[:column])  }.join(", ")})')").first["indexname"]
-          end
+          candidates[col_set] = execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(table)} (#{col_set.map { |c| quote_ident(c[:column])  }.join(", ")})')").first["indexname"]
         end
       end
     end

data/lib/dexter/log_parser.rb

@@ -12,7 +12,7 @@ module Dexter
       active_line = nil
       duration = nil
 
-      each_line do |line|
+      @logfile.each_line do |line|
         if active_line
           if line.include?(LINE_SEPERATOR)
             process_entry(active_line, duration)
@@ -32,12 +32,6 @@ module Dexter
 
     private
 
-    def each_line
-      @logfile.each_line do |line|
-        yield line
-      end
-    end
-
     def process_entry(query, duration)
       @collector.add(query, duration)
     end

data/lib/dexter/logging.rb

@@ -1,7 +1,7 @@
 module Dexter
   module Logging
-    def log(message)
-      puts "#{Time.now.iso8601} #{message}" unless $log_level == "error"
+    def log(message = "")
+      puts message unless $log_level == "error"
     end
   end
 end

data/lib/dexter/query.rb

@@ -1,7 +1,7 @@
 module Dexter
   class Query
     attr_reader :statement, :fingerprint, :plans
-    attr_accessor :missing_tables, :new_cost
+    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes
 
     def initialize(statement, fingerprint = nil)
       @statement = statement

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-28 00:00:00.000000000 Z
+date: 2017-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop