pgdexter 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f136be602609b6abb8f11ef02fa7cf18a8db33a
-  data.tar.gz: c58324b44d32b867df2a5a127a70b4d713e2dfa2
+  metadata.gz: d08be58d7de049df2c11f33211c0ada165f4a363
+  data.tar.gz: ce17e302b12f3b7babe38de9618b76bab982b9d9
 SHA512:
-  metadata.gz: 8f39e8a9794a937fe79a637f46e76ee71eba53e9bf3b60a734ad6af270a9236216d5182440586bde0762fd05d48be11c02bb3e620039ec02e2acce1633b81d36
-  data.tar.gz: b870ca6f00a6bc2f6d76c49d6805d0910d4053457ca5c11ccb666de863406fcd07acf37fdf7f23fb4d89f0df6795724fe74fca601738aa47f6832edbafa03a86
+  metadata.gz: 7e0974f0f8c39ec8039749a75a3fb61f7613c6931db15f191eba2dda2a794cf57e8875f6aff4465474a9c608727deef08fd7a5833e16ba96e2d1d5a02e3e7f7b
+  data.tar.gz: 73668e8c819b4334888bec707db491453ab2cc03a39e189e451738bb7606b8fe329912e2520d6e2b6b7aeb9634a8e2200b267887257c26428bf453fde74cc54e

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.4
+
+- Added support for multicolumn indexes
+
 ## 0.1.3
 
 - Fixed error with non-lowercase columns

data/README.md

@@ -13,12 +13,12 @@ wget https://github.com/dalibo/hypopg/archive/1.0.0.tar.gz
 tar xf 1.0.0.tar.gz
 cd hypopg-1.0.0
 make
-make install
+make install # may need sudo
 ```
 
 > Note: If you have issues, make sure `postgresql-server-dev-*` is installed.
 
-Enable logging for slow queries.
+Enable logging for slow queries in your Postgres config file.
 
 ```ini
 log_min_duration_statement = 10 # ms
@@ -30,6 +30,8 @@ And install the command line tool with:
 gem install pgdexter
 ```
 
+The command line tool is also available as a [Linux package](guides/Linux.md).
+
 ## How to Use
 
 Dexter needs a connection to your database and a log file to process.
@@ -60,13 +62,21 @@ To be safe, Dexter will not create indexes unless you pass the `--create` flag.
 2017-06-25T17:52:37+00:00 Index created: 15243 ms
 ```
 
+## Single Statement Mode
+
+You can also pass a single statement with:
+
+```sh
+dexter <database-url> -s "SELECT * FROM ..."
+```
+
 ## Options
 
 Name | Description | Default
 --- | --- | ---
 exclude | prevent specific tables from being indexed | None
 interval | time to wait between processing queries, in seconds | 60
-log-level | `debug` gives additional info for suggested indexes<br />`debug2` gives additional info for all processed queries | info
+log-level | `debug` gives additional info for suggested indexes<br />`debug2` gives additional info for processed queries | info
 log-sql | log SQL statements executed | false
 min-time | only process queries consuming a min amount of DB time, in minutes | 0
 

data/guides/Linux.md

@@ -0,0 +1,59 @@
+# Linux Packages
+
+Distributions
+
+- [Ubuntu 16.04 (Xenial)](#ubuntu-1604-xenial)
+- [Ubuntu 14.04 (Trusty)](#ubuntu-1404-trusty)
+- [Debian 8 (Jesse)](#debian-8-jesse)
+- [CentOS / RHEL 7](#centos--rhel-7)
+- [SUSE Linux Enterprise Server 12](#suse-linux-enterprise-server-12)
+
+### Ubuntu 16.04 (Xenial)
+
+```sh
+wget -qO - https://deb.packager.io/key | sudo apt-key add -
+echo "deb https://deb.packager.io/gh/pghero/dexter xenial master" | sudo tee /etc/apt/sources.list.d/dexter.list
+sudo apt-get update
+sudo apt-get -y install dexter
+```
+
+### Ubuntu 14.04 (Trusty)
+
+```sh
+wget -qO - https://deb.packager.io/key | sudo apt-key add -
+echo "deb https://deb.packager.io/gh/pghero/dexter trusty master" | sudo tee /etc/apt/sources.list.d/dexter.list
+sudo apt-get update
+sudo apt-get install dexter
+```
+
+### Debian 8 (Jesse)
+
+```sh
+wget -qO - https://deb.packager.io/key | sudo apt-key add -
+echo "deb https://deb.packager.io/gh/pghero/dexter jessie master" | sudo tee /etc/apt/sources.list.d/dexter.list
+sudo apt-get update
+sudo apt-get install dexter
+```
+
+### CentOS / RHEL 7
+
+```sh
+sudo rpm --import https://rpm.packager.io/key
+echo "[dexter]
+name=Repository for pghero/dexter application.
+baseurl=https://rpm.packager.io/gh/pghero/dexter/centos7/master
+enabled=1" | sudo tee /etc/yum.repos.d/dexter.repo
+sudo yum install dexter
+```
+
+### SUSE Linux Enterprise Server 12
+
+```sh
+sudo rpm --import https://rpm.packager.io/key
+sudo zypper addrepo "https://rpm.packager.io/gh/pghero/dexter/sles12/master" "dexter"
+sudo zypper install dexter
+```
+
+## Credits
+
+:heart: Made possible by [Packager](https://packager.io/)

data/lib/dexter/client.rb

@@ -31,6 +31,7 @@ Options:)
         o.array "--exclude", "prevent specific tables from being indexed"
         o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
         o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
+        o.boolean "--log-explain", "log explain", default: false, help: false
         o.string "--log-level", "log level", default: "info"
         o.boolean "--log-sql", "log sql", default: false
         o.string "-s", "--statement", "process a single statement"

data/lib/dexter/indexer.rb

@@ -8,6 +8,7 @@ module Dexter
       @log_level = options[:log_level]
       @exclude_tables = options[:exclude]
       @log_sql = options[:log_sql]
+      @log_explain = options[:log_explain]
 
       create_extension
     end
@@ -31,13 +32,10 @@ module Dexter
       # analyze tables if needed
       analyze_tables(tables) if tables.any?
 
-      # get initial costs for queries
-      calculate_initial_cost(queries.reject(&:missing_tables))
-
-      # create hypothetical indexes
-      candidates = tables.any? ? create_hypothetical_indexes(tables) : {}
+      # create hypothetical indexes and explain queries
+      candidates = tables.any? ? create_hypothetical_indexes(queries.reject(&:missing_tables), tables) : {}
 
-      # get new costs and see if new indexes were used
+      # see if new indexes were used and meet bar
       new_indexes = determine_indexes(queries, candidates)
 
       # display and create new indexes
@@ -47,18 +45,18 @@ module Dexter
     private
 
     def create_extension
-      select_all("SET client_min_messages = warning")
-      select_all("CREATE EXTENSION IF NOT EXISTS hypopg")
+      execute("SET client_min_messages = warning")
+      execute("CREATE EXTENSION IF NOT EXISTS hypopg")
     end
 
     def reset_hypothetical_indexes
-      select_all("SELECT hypopg_reset()")
+      execute("SELECT hypopg_reset()")
     end
 
     def analyze_tables(tables)
       tables = tables.to_a.sort
 
-      analyze_stats = select_all <<-SQL
+      analyze_stats = execute <<-SQL
         SELECT
           schemaname AS schema,
           relname AS table,
@@ -79,22 +77,33 @@ module Dexter
         if !last_analyzed[table] || last_analyzed[table] < Time.now - 3600
           statement = "ANALYZE #{quote_ident(table)}"
           log "Running analyze: #{statement}"
-          select_all(statement)
+          execute(statement)
         end
       end
     end
 
-    def calculate_initial_cost(queries)
+    def calculate_plan(queries)
       queries.each do |query|
         begin
-          query.initial_cost = plan(query.statement)["Total Cost"]
+          query.plans << plan(query.statement)
+          if @log_explain
+            log "Explaining query"
+            puts
+            # Pass format to prevent ANALYZE
+            puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query.statement)}").map { |r| r["QUERY PLAN"] }.join("\n")
+            puts
+          end
         rescue PG::Error
           # do nothing
         end
       end
     end
 
-    def create_hypothetical_indexes(tables)
+    def create_hypothetical_indexes(queries, tables)
+      # get initial costs for queries
+      calculate_plan(queries)
+      explainable_queries = queries.select(&:explainable?)
+
       # get existing indexes
       index_set = Set.new
       indexes(tables).each do |index|
@@ -104,13 +113,20 @@ module Dexter
 
       # create hypothetical indexes
       candidates = {}
-      columns(tables).each do |col|
-        unless index_set.include?([col[:table], [col[:column]]])
-          unless ["json", "jsonb"].include?(col[:type])
-            candidates[col] = select_all("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(col[:table])} (#{[col[:column]].map { |c| quote_ident(c)  }.join(", ")})')").first["indexname"]
-          end
-        end
-      end
+      columns_by_table = columns(tables).group_by { |c| c[:table] }
+
+      # create single column indexes
+      create_hypothetical_indexes_helper(columns_by_table, 1, index_set, candidates)
+
+      # get next round of costs
+      calculate_plan(explainable_queries)
+
+      # create multicolumn indexes
+      create_hypothetical_indexes_helper(columns_by_table, 2, index_set, candidates)
+
+      # get next round of costs
+      calculate_plan(explainable_queries)
+
       candidates
     end
 
@@ -118,17 +134,23 @@ module Dexter
       new_indexes = {}
 
       queries.each do |query|
-        if query.initial_cost
-          new_plan = plan(query.statement)
-          query.new_cost = new_plan["Total Cost"]
-          cost_savings = query.new_cost < query.initial_cost * 0.5
+        if query.explainable?
+          new_cost, new_cost2 = query.costs[1..2]
+
+          cost_savings = new_cost < query.initial_cost * 0.5
+          # set high bar for multicolumn indexes
+          cost_savings2 = new_cost > 100 && new_cost2 < new_cost * 0.5
+
+          query.new_cost = cost_savings2 ? new_cost2 : new_cost
 
           query_indexes = []
-          candidates.each do |col, index_name|
-            if new_plan.inspect.include?(index_name)
+          candidates.each do |col_set, index_name|
+            key = cost_savings2 ? 2 : 1
+
+            if query.plans[key].inspect.include?(index_name)
               index = {
-                table: col[:table],
-                columns: [col[:column]]
+                table: col_set[0][:table],
+                columns: col_set.map { |c| c[:column] }
               }
               query_indexes << index
 
@@ -142,12 +164,12 @@ module Dexter
 
         if @log_level == "debug2"
           log "Processed #{query.fingerprint}"
-          if query.initial_cost
+          if query.explainable?
             log "Cost: #{query.initial_cost} -> #{query.new_cost}"
 
             if query_indexes.any?
               log "Indexes: #{query_indexes.map { |i| "#{i[:table]} (#{i[:columns].join(", ")})" }.join(", ")}"
-              log "Need 50% cost savings to suggest index" unless cost_savings
+              log "Need 50% cost savings to suggest index" unless cost_savings || cost_savings2
             else
               log "Indexes: None"
             end
@@ -195,7 +217,7 @@ module Dexter
             statement = "CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})"
             log "Creating index: #{statement}"
             started_at = Time.now
-            select_all(statement)
+            execute(statement)
             log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
           end
         end
@@ -223,7 +245,7 @@ module Dexter
       abort "Bad database url"
     end
 
-    def select_all(query)
+    def execute(query)
       # use exec_params instead of exec for security
       #
       # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
@@ -238,11 +260,23 @@ module Dexter
 
     def plan(query)
       # strip semi-colons as another measure of defense
-      JSON.parse(select_all("EXPLAIN (FORMAT JSON) #{query.gsub(";", "")}").first["QUERY PLAN"]).first["Plan"]
+      JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}").first["QUERY PLAN"]).first["Plan"]
+    end
+
+    # TODO for multicolumn indexes, use ordering
+    def create_hypothetical_indexes_helper(columns_by_table, n, index_set, candidates)
+      columns_by_table.each do |table, cols|
+        # no reason to use btree index for json columns
+        cols.reject { |c| ["json", "jsonb"].include?(c[:type]) }.permutation(n) do |col_set|
+          if !index_set.include?([table, col_set.map { |col| col[:column] }])
+            candidates[col_set] = execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(table)} (#{col_set.map { |c| quote_ident(c[:column])  }.join(", ")})')").first["indexname"]
+          end
+        end
+      end
     end
 
     def database_tables
-      result = select_all <<-SQL
+      result = execute <<-SQL
         SELECT
           table_name
         FROM
@@ -259,7 +293,7 @@ module Dexter
     end
 
     def columns(tables)
-      columns = select_all <<-SQL
+      columns = execute <<-SQL
         SELECT
           table_name,
           column_name,
@@ -269,13 +303,15 @@ module Dexter
         WHERE
           table_schema = 'public' AND
           table_name IN (#{tables.map { |t| quote(t) }.join(", ")})
+        ORDER BY
+          1, 2
       SQL
 
       columns.map { |v| {table: v["table_name"], column: v["column_name"], type: v["data_type"]} }
     end
 
     def indexes(tables)
-      select_all(<<-SQL
+      execute(<<-SQL
         SELECT
           schemaname AS schema,
           t.relname AS table,
@@ -331,5 +367,9 @@ module Dexter
     def squish(str)
       str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
     end
+
+    def safe_statement(statement)
+      statement.gsub(";", "")
+    end
   end
 end

data/lib/dexter/query.rb

@@ -1,15 +1,28 @@
 module Dexter
   class Query
-    attr_reader :statement, :fingerprint
-    attr_accessor :initial_cost, :new_cost, :missing_tables
+    attr_reader :statement, :fingerprint, :plans
+    attr_accessor :missing_tables, :new_cost
 
     def initialize(statement, fingerprint)
       @statement = statement
       @fingerprint = fingerprint
+      @plans = []
     end
 
     def tables
       @tables ||= PgQuery.parse(statement).tables rescue []
     end
+
+    def explainable?
+      plans.any?
+    end
+
+    def costs
+      plans.map { |plan| plan["Total Cost"] }
+    end
+
+    def initial_cost
+      costs[0]
+    end
   end
 end

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-01 00:00:00.000000000 Z
+date: 2017-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -109,6 +109,7 @@ files:
 - README.md
 - Rakefile
 - exe/dexter
+- guides/Linux.md
 - lib/dexter.rb
 - lib/dexter/client.rb
 - lib/dexter/collector.rb