pgdexter 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fbc105f249fc27a10fcd47fa16ba9d732f0cd933
-  data.tar.gz: 20929a8fd1d268194b9bc2c7bab71130a2eabc43
+  metadata.gz: 496685bbacfac7387182be3ebda3515a492723cc
+  data.tar.gz: b99548fccab8337b9ff5765a6182472abba9bb06
 SHA512:
-  metadata.gz: b01d8e32db41bdf7fa0dd2412b4b490318d8574fbfba5ed0357f1aa9f3e8dcc2e9a0fbef9ee80376678c184f8840a5bf5e52fef5b325cdbffb2b31106d5c2838
-  data.tar.gz: 2e9808dba0e161eb184ddc180ee915f0c49097225b2539a11916eaf5dd32e9bedce8ef07763391f758d6f4627258c90e6af4f7c158b473323d398050e64a9f75
+  metadata.gz: 3f9b30e7355c26cf7084b8deee9735206da484cd9f4725a5c1f751db5808dc32d66954341c5be3ba46cd783fa2e360e2b9c32ab0667aa2372213ebca695b29ff
+  data.tar.gz: 41346b8ed9906e24c68847c26bc9ffe54176eb0e74dc771138ec266ebccfca3e0d18ebd37223b73f1f2b8a9f512320502e4ad9597982bc253846881ea7e93e8d

data/.travis.yml

@@ -0,0 +1,18 @@
+language: ruby
+rvm: 2.4.1
+cache: bundler
+script: bundle exec rake test
+addons:
+  postgresql: "9.6"
+before_script:
+  - sudo apt-get install postgresql-server-dev-9.6
+  - wget https://github.com/dalibo/hypopg/archive/1.0.0.tar.gz
+  - tar xf 1.0.0.tar.gz
+  - cd hypopg-1.0.0
+  - make
+  - sudo make install
+  - psql -c 'create database dexter_test;' -U postgres
+notifications:
+  email:
+    on_success: never
+    on_failure: change

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.6
+
+- Significant performance improvements
+- Added `--include` option
+
 ## 0.1.5
 
 - Added support for non-`SELECT` queries

data/README.md

@@ -4,13 +4,14 @@ The automatic indexer for Postgres
 
 [Read about how it works](https://medium.com/@ankane/introducing-dexter-the-automatic-indexer-for-postgres-5f8fa8b28f27)
 
+[![Build Status](https://travis-ci.org/ankane/dexter.svg?branch=master)](https://travis-ci.org/ankane/dexter)
+
 ## Installation
 
 First, install [HypoPG](https://github.com/dalibo/hypopg) on your database server. This doesn’t require a restart.
 
 ```sh
-wget https://github.com/dalibo/hypopg/archive/1.0.0.tar.gz
-tar xf 1.0.0.tar.gz
+curl -L https://github.com/dalibo/hypopg/archive/1.0.0.tar.gz | tar -x
 cd hypopg-1.0.0
 make
 make install # may need sudo

data/lib/dexter/client.rb

@@ -30,6 +30,7 @@ module Dexter
 Options:)
         o.boolean "--create", "create indexes", default: false
         o.array "--exclude", "prevent specific tables from being indexed"
+        o.string "--include", "only include specific tables"
         o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
         o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
         o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false

data/lib/dexter/indexer.rb

@@ -7,11 +7,13 @@ module Dexter
       @create = options[:create]
       @log_level = options[:log_level]
       @exclude_tables = options[:exclude]
+      @include_tables = Array(options[:include].split(",")) if options[:include]
       @log_sql = options[:log_sql]
       @log_explain = options[:log_explain]
       @min_time = options[:min_time] || 0
 
       create_extension unless extension_exists?
+      execute("SET lock_timeout = '5s'")
     end
 
     def process_stat_statements
@@ -30,6 +32,10 @@ module Dexter
         query.missing_tables = !query.tables.all? { |t| tables.include?(t) }
       end
 
+      if @include_tables
+        tables = Set.new(tables.to_a & @include_tables)
+      end
+
       # exclude user specified tables
       # TODO exclude write-heavy tables
       @exclude_tables.each do |table|
@@ -43,7 +49,7 @@ module Dexter
       candidates = tables.any? ? create_hypothetical_indexes(queries.reject(&:missing_tables), tables) : {}
 
       # see if new indexes were used and meet bar
-      new_indexes = determine_indexes(queries, candidates)
+      new_indexes = determine_indexes(queries, candidates, tables)
 
       # display and create new indexes
       show_and_create_indexes(new_indexes)
@@ -55,6 +61,8 @@ module Dexter
       execute("SET client_min_messages = warning")
       begin
         execute("CREATE EXTENSION IF NOT EXISTS hypopg")
+      rescue PG::UndefinedFile
+        abort "Install HypoPG first: https://github.com/ankane/dexter#installation"
       rescue PG::InsufficientPrivilege
         abort "Use a superuser to run: CREATE EXTENSION hypopg"
       end
@@ -115,41 +123,85 @@ module Dexter
     end
 
     def create_hypothetical_indexes(queries, tables)
+      candidates = {}
+
       # get initial costs for queries
       calculate_plan(queries)
-      explainable_queries = queries.select(&:explainable?)
+      explainable_queries = queries.select { |q| q.explainable? && q.high_cost? }
 
-      # get existing indexes
-      index_set = Set.new
-      indexes(tables).each do |index|
-        # TODO make sure btree
-        index_set << [index["table"], index["columns"]]
-      end
+      # filter tables for performance
+      tables = Set.new(explainable_queries.flat_map(&:tables))
 
-      # create hypothetical indexes
-      candidates = {}
-      columns_by_table = columns(tables).group_by { |c| c[:table] }
+      if tables.any?
+        # get existing indexes
+        index_set = Set.new
+        indexes(tables).each do |index|
+          # TODO make sure btree
+          index_set << [index["table"], index["columns"]]
+        end
+
+        # since every set of multi-column indexes are expensive
+        # try to parse out columns
+        possible_columns = Set.new
+        explainable_queries.each do |query|
+          find_columns(query.tree).each do |col|
+            last_col = col["fields"].last
+            if last_col["String"]
+              possible_columns << last_col["String"]["str"]
+            end
+          end
+        end
+
+        # create hypothetical indexes
+        columns_by_table = columns(tables).select { |c| possible_columns.include?(c[:column]) }.group_by { |c| c[:table] }
 
-      # create single column indexes
-      create_hypothetical_indexes_helper(columns_by_table, 1, index_set, candidates)
+        # create single column indexes
+        create_hypothetical_indexes_helper(columns_by_table, 1, index_set, candidates)
 
-      # get next round of costs
-      calculate_plan(explainable_queries)
+        # get next round of costs
+        calculate_plan(explainable_queries)
 
-      # create multicolumn indexes
-      create_hypothetical_indexes_helper(columns_by_table, 2, index_set, candidates)
+        # create multicolumn indexes
+        create_hypothetical_indexes_helper(columns_by_table, 2, index_set, candidates)
 
-      # get next round of costs
-      calculate_plan(explainable_queries)
+        # get next round of costs
+        calculate_plan(explainable_queries)
+      end
 
       candidates
     end
 
-    def determine_indexes(queries, candidates)
+    def find_columns(plan)
+      find_by_key(plan, "ColumnRef")
+    end
+
+    def find_indexes(plan)
+      find_by_key(plan, "Index Name")
+    end
+
+    def find_by_key(plan, key)
+      indexes = []
+      case plan
+      when Hash
+        plan.each do |k, v|
+          if k == key
+            indexes << v
+          else
+            indexes.concat(find_by_key(v, key))
+          end
+        end
+      when Array
+        indexes.concat(plan.flat_map { |v| find_by_key(v, key) })
+      end
+      indexes
+    end
+
+    def determine_indexes(queries, candidates, tables)
       new_indexes = {}
+      index_name_to_columns = candidates.invert
 
       queries.each do |query|
-        if query.explainable?
+        if query.explainable? && query.high_cost?
           new_cost, new_cost2 = query.costs[1..2]
 
           cost_savings = new_cost < query.initial_cost * 0.5
@@ -159,10 +211,13 @@ module Dexter
           query.new_cost = cost_savings2 ? new_cost2 : new_cost
 
           query_indexes = []
-          candidates.each do |col_set, index_name|
-            key = cost_savings2 ? 2 : 1
+          key = cost_savings2 ? 2 : 1
+          indexes = find_indexes(query.plans[key]).uniq.sort
+
+          indexes.each do |index_name|
+            col_set = index_name_to_columns[index_name]
 
-            if query.plans[key].inspect.include?(index_name)
+            if col_set
               index = {
                 table: col_set[0][:table],
                 columns: col_set.map { |c| c[:column] }
@@ -179,7 +234,11 @@ module Dexter
 
         if @log_level == "debug2"
           log "Processed #{query.fingerprint}"
-          if query.explainable?
+          if tables.empty?
+            log "No candidate tables for indexes"
+          elsif query.explainable? && !query.high_cost?
+            log "Low initial cost: #{query.initial_cost}"
+          elsif query.explainable?
             log "Cost: #{query.initial_cost} -> #{query.new_cost}"
 
             if query_indexes.any?
@@ -233,8 +292,12 @@ module Dexter
                 statement = "CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})"
                 log "Creating index: #{statement}"
                 started_at = Time.now
-                execute(statement)
-                log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
+                begin
+                  execute(statement)
+                  log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
+                rescue PG::LockNotAvailable => e
+                  log "Could not acquire lock: #{index[:table]}"
+                end
               end
             end
           end

data/lib/dexter/log_parser.rb

@@ -6,6 +6,8 @@ module Dexter
     def initialize(logfile, collector)
       @logfile = logfile
       @collector = collector
+
+      abort "Log file not found" unless File.exist?(logfile)
     end
 
     def perform
@@ -38,12 +40,8 @@ module Dexter
           yield line
         end
       else
-        begin
-          File.foreach(@logfile) do |line|
-            yield line
-          end
-        rescue Errno::ENOENT
-          abort "Log file not found"
+        File.foreach(@logfile) do |line|
+          yield line
         end
       end
     end

data/lib/dexter/processor.rb

@@ -3,8 +3,6 @@ module Dexter
     include Logging
 
     def initialize(database_url, logfile, options)
-      log "Started"
-
       @logfile = logfile
 
       @collector = Collector.new(min_time: options[:min_time])
@@ -16,6 +14,8 @@ module Dexter
 
       @mutex = Mutex.new
       @last_checked_at = {}
+
+      log "Started"
     end
 
     def perform
@@ -24,7 +24,11 @@ module Dexter
         Thread.new do
           sleep(@starting_interval)
           loop do
-            process_queries
+            begin
+              process_queries
+            rescue PG::ServerError => e
+              log "ERROR: #{e.class.name}: #{e.message}"
+            end
             sleep(@interval)
           end
         end

data/lib/dexter/query.rb

@@ -13,7 +13,11 @@ module Dexter
     end
 
     def tables
-      @tables ||= PgQuery.parse(statement).tables rescue []
+      @tables ||= parse ? parse.tables : []
+    end
+
+    def tree
+      parse.tree
     end
 
     def explainable?
@@ -27,5 +31,18 @@ module Dexter
     def initial_cost
       costs[0]
     end
+
+    def high_cost?
+      initial_cost && initial_cost >= 100
+    end
+
+    private
+
+    def parse
+      unless defined?(@parse)
+        @parse = PgQuery.parse(statement) rescue nil
+      end
+      @parse
+    end
   end
 end

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-15 00:00:00.000000000 Z
+date: 2017-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -103,6 +103,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt