pgdexter 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 714767248afe28ad9e354ebb6485b34a8d6aadd0
-  data.tar.gz: 545eaaea1df0312049f4cecff3fe9b03b80a6cd9
+  metadata.gz: 14bc122b136301535793c29b19336a2d0614f640
+  data.tar.gz: 624c8c6ae5aabd8a5e2150887aa83efe6e67f9b4
 SHA512:
-  metadata.gz: '096684887c5d5a48a2df74a2dae6cf70bc2b5e98dd5dc6d7d6d8b583bfb76193ce86166972d9553f89452f16629a8576a30f2627fd8d0b16a9672ce76a316835'
-  data.tar.gz: d128734845414672a3f470f138c6a422dc2b24024a66866e50524fe3bb3e949bd4a899305a93f71fb14af3b56820f66e20a00d5ba5788cd0c88bd0afaa064e32
+  metadata.gz: 44224c687d8590c0441d587b432fde3b85cef77728980059d497ade992cd953e8f8445747de61c984d559f55d74461ac284c6c8423c0f1f8b08e439e361c141d
+  data.tar.gz: 42ec40ec7527dfc3093853ec013c5b30bacbaee5cbe8d27a1530649bfbdc16c639d9d93d65e72b062d53b33bff1d0a1bec67e971d283f5d5a2e6fea7d884fccd

data/.travis.yml

@@ -6,9 +6,8 @@ addons:
   postgresql: "9.6"
 before_script:
   - sudo apt-get install postgresql-server-dev-9.6
-  - wget https://github.com/dalibo/hypopg/archive/1.0.0.tar.gz
-  - tar xf 1.0.0.tar.gz
-  - cd hypopg-1.0.0
+  - git clone https://github.com/dalibo/hypopg.git
+  - cd hypopg
   - make
   - sudo make install
   - psql -c 'create database dexter_test;' -U postgres

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.3
+
+- Added support for views and materialized views
+- Better handle case when multiple indexes are found for a query
+- Added `--min-cost-savings-pct` option
+
 ## 0.3.2
 
 - Fixed parsing issue with named prepared statements

data/README.md

@@ -110,6 +110,8 @@ or use the [pg_stat_statements](https://www.postgresql.org/docs/current/static/p
 dexter <connection-options> --pg-stat-statements
 ```
 
+> Note: Logs are highly preferred over pg_stat_statements, as pg_stat_statements often doesn’t store enough information to optimize queries.
+
 ### Collection Options
 
 To prevent one-off queries from being indexed, specify a minimum number of calls before a query is considered for indexing

data/guides/Linux.md

@@ -4,6 +4,7 @@ Distributions
 
 - [Ubuntu 16.04 (Xenial)](#ubuntu-1604-xenial)
 - [Ubuntu 14.04 (Trusty)](#ubuntu-1404-trusty)
+- [Debian 9 (Stretch)](#debian-9-stretch)
 - [Debian 8 (Jesse)](#debian-8-jesse)
 - [CentOS / RHEL 7](#centos--rhel-7)
 - [SUSE Linux Enterprise Server 12](#suse-linux-enterprise-server-12)
@@ -28,6 +29,16 @@ sudo apt-get update
 sudo apt-get install dexter
 ```
 
+### Debian 9 (Stretch)
+
+```sh
+wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
+sudo wget -O /etc/apt/sources.list.d/dexter.list \
+  https://dl.packager.io/srv/pghero/dexter/master/installer/debian/9.repo
+sudo apt-get update
+sudo apt-get install dexter
+```
+
 ### Debian 8 (Jesse)
 
 ```sh

data/lib/dexter/client.rb

@@ -43,6 +43,7 @@ Options:)
         o.boolean "--log-sql", "log sql", default: false
         o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
         o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
+        o.integer "--min-cost-savings-pct", default: 50, help: false
         o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
         o.string "-s", "--statement", "process a single statement"
         # separator must go here to show up correctly - slop bug?

data/lib/dexter/indexer.rb

@@ -12,6 +12,7 @@ module Dexter
       @min_time = options[:min_time] || 0
       @min_calls = options[:min_calls] || 0
       @analyze = options[:analyze]
+      @min_cost_savings_pct = options[:min_cost_savings_pct].to_i
       @options = options
 
       create_extension unless extension_exists?
@@ -28,7 +29,7 @@ module Dexter
       # reset hypothetical indexes
       reset_hypothetical_indexes
 
-      tables = Set.new(database_tables)
+      tables = Set.new(database_tables + materialized_views)
 
       # map tables without schema to schema
       no_schema_tables = {}
@@ -37,11 +38,28 @@ module Dexter
         no_schema_tables[group] = t2.sort_by { |t| [search_path_index[t.split(".")[0]] || 1000000, t] }[0]
       end
 
+      # add tables from views
+      view_tables = database_view_tables
+      view_tables.each do |v, vt|
+        view_tables[v] = vt.map { |t| no_schema_tables[t] || t }
+      end
+
+      # fully resolve tables
+      # make sure no views in result
+      view_tables.each do |v, vt|
+        view_tables[v] = vt.flat_map { |t| view_tables[t] || [t] }.uniq
+      end
+
       # filter queries from other databases and system tables
       queries.each do |query|
         # add schema to table if needed
         query.tables = query.tables.map { |t| no_schema_tables[t] || t }
 
+        # substitute view tables
+        new_tables = query.tables.flat_map { |t| view_tables[t] || [t] }.uniq
+        query.tables_from_views = new_tables - query.tables
+        query.tables = new_tables
+
         # check for missing tables
         query.missing_tables = !query.tables.all? { |t| tables.include?(t) }
       end
@@ -166,6 +184,7 @@ module Dexter
 
       # filter tables for performance
       tables = Set.new(explainable_queries.flat_map(&:tables))
+      tables_from_views = Set.new(explainable_queries.flat_map(&:tables_from_views))
 
       if tables.any?
         # since every set of multi-column indexes are expensive
@@ -182,7 +201,8 @@ module Dexter
         end
 
         # create hypothetical indexes
-        columns_by_table = columns(tables).select { |c| possible_columns.include?(c[:column]) }.group_by { |c| c[:table] }
+        # use all columns in tables from views
+        columns_by_table = columns(tables).select { |c| possible_columns.include?(c[:column]) || tables_from_views.include?(c[:table]) }.group_by { |c| c[:table] }
 
         # create single column indexes
         create_hypothetical_indexes_helper(columns_by_table, 1, candidates)
@@ -265,14 +285,16 @@ module Dexter
         end
       end
 
+      savings_ratio = (1 - @min_cost_savings_pct / 100.0)
+
       queries.each do |query|
         if query.explainable? && query.high_cost?
           new_cost, new_cost2 = query.costs[1..2]
 
-          cost_savings = new_cost < query.initial_cost * 0.5
+          cost_savings = new_cost < query.initial_cost * savings_ratio
 
           # set high bar for multicolumn indexes
-          cost_savings2 = new_cost > 100 && new_cost2 < new_cost * 0.5
+          cost_savings2 = new_cost > 100 && new_cost2 < new_cost * savings_ratio
 
           key = cost_savings2 ? 2 : 1
           query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[key], index_set)
@@ -283,10 +305,55 @@ module Dexter
             cost_savings2 = false
           end
 
-          # TODO if multiple indexes are found (for either single or multicolumn)
+          suggest_index = cost_savings || cost_savings2
+
+          cost_savings3 = false
+          new_cost3 = nil
+
+          # if multiple indexes are found (for either single or multicolumn)
           # determine the impact of each individually
-          # for now, be conservative and don't suggest if more than one index
-          suggest_index = (cost_savings || cost_savings2) && query_indexes.size == 1
+          # there may be a better single index that we're not considering
+          # that didn't get picked up by pass1 or pass2
+          # TODO clean this up
+          # TODO suggest more than one index from this if savings are there
+          if suggest_index && query_indexes.size > 1
+            winning_index = nil
+            winning_cost = nil
+            winning_plan = nil
+
+            query_indexes.each do |query_index|
+              reset_hypothetical_indexes
+              create_hypothetical_index(query_index[:table], query_index[:columns].map { |v| {column: v} })
+              plan3 = plan(query.statement)
+              cost3 = plan3["Total Cost"]
+
+              if !winning_cost || cost3 < winning_cost
+                winning_cost = cost3
+                winning_index = query_index
+                winning_plan = plan3
+              end
+            end
+
+            query.plans << winning_plan
+
+            # duplicated from above
+            # TODO DRY
+            use_winning =
+              if cost_savings2
+                new_cost > 100 && winning_cost < new_cost * savings_ratio
+              else
+                winning_cost < query.initial_cost * savings_ratio
+              end
+
+            if use_winning
+              query_indexes = [winning_index]
+              cost_savings3 = true
+              new_cost3 = winning_cost
+              query.pass3_indexes = query_indexes
+            else
+              suggest_index = false
+            end
+          end
 
           if suggest_index
             query_indexes.each do |index|
@@ -299,7 +366,7 @@ module Dexter
           query.suggest_index = suggest_index
           query.new_cost =
             if suggest_index
-              cost_savings2 ? new_cost2 : new_cost
+              cost_savings3 ? new_cost3 : (cost_savings2 ? new_cost2 : new_cost)
             else
               query.initial_cost
             end
@@ -368,9 +435,12 @@ module Dexter
             log "Start: #{query.costs[0]}"
             log "Pass1: #{query.costs[1]} : #{log_indexes(query.pass1_indexes || [])}"
             log "Pass2: #{query.costs[2]} : #{log_indexes(query.pass2_indexes || [])}"
+            if query.costs[3]
+              log "Pass3: #{query.costs[3]} : #{log_indexes(query.pass3_indexes || [])}"
+            end
             log "Final: #{query.new_cost} : #{log_indexes(query.suggest_index ? query_indexes : [])}"
             if query_indexes.size == 1 && !query.suggest_index
-              log "Need 50% cost savings to suggest index"
+              log "Need #{@min_cost_savings_pct}% cost savings to suggest index"
             end
           else
             log "Could not run explain"
@@ -449,11 +519,15 @@ module Dexter
       columns_by_table.each do |table, cols|
         # no reason to use btree index for json columns
         cols.reject { |c| ["json", "jsonb"].include?(c[:type]) }.permutation(n) do |col_set|
-          candidates[col_set] = execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(table)} (#{col_set.map { |c| quote_ident(c[:column])  }.join(", ")})')").first["indexname"]
+          candidates[col_set] = create_hypothetical_index(table, col_set)
         end
       end
     end
 
+    def create_hypothetical_index(table, col_set)
+      execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(table)} (#{col_set.map { |c| quote_ident(c[:column])  }.join(", ")})')").first["indexname"]
+    end
+
     def database_tables
       result = execute <<-SQL
         SELECT
@@ -466,6 +540,43 @@ module Dexter
       result.map { |r| r["table_name"] }
     end
 
+    def materialized_views
+      if server_version_num >= 90300
+        result = execute <<-SQL
+          SELECT
+            schemaname || '.' || matviewname AS table_name
+          FROM
+            pg_matviews
+        SQL
+        result.map { |r| r["table_name"] }
+      else
+        []
+      end
+    end
+
+    def server_version_num
+      execute("SHOW server_version_num").first["server_version_num"].to_i
+    end
+
+    def database_view_tables
+      result = execute <<-SQL
+        SELECT
+          schemaname || '.' || viewname AS table_name,
+          definition
+        FROM
+          pg_views
+        WHERE
+          schemaname NOT IN ('information_schema', 'pg_catalog')
+      SQL
+
+      view_tables = {}
+      result.each do |row|
+        view_tables[row["table_name"]] = PgQuery.parse(row["definition"]).tables
+      end
+
+      view_tables
+    end
+
     def stat_statements
       result = execute <<-SQL
         SELECT
@@ -515,13 +626,15 @@ module Dexter
     def columns(tables)
       columns = execute <<-SQL
         SELECT
-          table_schema || '.' || table_name AS table_name,
-          column_name,
-          data_type
-        FROM
-          information_schema.columns
-        WHERE
-          table_schema || '.' || table_name IN (#{tables.map { |t| quote(t) }.join(", ")})
+          s.nspname || '.' || t.relname AS table_name,
+          a.attname AS column_name,
+          pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type
+        FROM pg_attribute a
+          JOIN pg_class t on a.attrelid = t.oid
+          JOIN pg_namespace s on t.relnamespace = s.oid
+        WHERE a.attnum > 0
+          AND NOT a.attisdropped
+          AND s.nspname || '.' || t.relname IN (#{tables.map { |t| quote(t) }.join(", ")})
         ORDER BY
           1, 2
       SQL

data/lib/dexter/query.rb

@@ -2,7 +2,7 @@ module Dexter
   class Query
     attr_reader :statement, :fingerprint, :plans
     attr_writer :tables
-    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :candidate_tables
+    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views
 
     def initialize(statement, fingerprint = nil)
       @statement = statement
@@ -11,6 +11,7 @@ module Dexter
       end
       @fingerprint = fingerprint
       @plans = []
+      @tables_from_views = []
     end
 
     def tables

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.3.2"
+  VERSION = "0.3.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-05 00:00:00.000000000 Z
+date: 2018-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop