pgdexter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 12871a558deb7cd0cb5067476415a34164d5d906
+  data.tar.gz: 6a293f4992bacb524a37bd7f7882851d364f6080
+SHA512:
+  metadata.gz: 846f1efba33eba3144e0918348068fc34fefe1ef8dfb62671231ccb01a836b2274b6ac6b43737484d43e0057556898b544aa12142df27860a881b4b9fa43b973
+  data.tar.gz: 676ad00ea2529681fbfe09e9982a30e49960a03f3dd25733ba206e2fed6bb0708789ec543acb193343b9ec0b31c0ecd12fd4f9d1adbbd9393324acd33322864c

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in dexter.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2017 Andrew Kane
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+# Dexter
+
+An automatic indexer for Postgres
+
+## Installation
+
+First, install [HypoPG](https://github.com/dalibo/hypopg) on your database server. This doesn’t require a restart.
+
+```sh
+wget https://github.com/dalibo/hypopg/archive/1.0.0.tar.gz
+tar xf 1.0.0.tar.gz
+cd hypopg-1.0.0
+make
+make install
+```
+
+> Note: If you have issues, make sure `postgresql-server-dev-*` is installed.
+
+Enable logging for slow queries.
+
+```ini
+log_min_duration_statement = 10 # ms
+```
+
+And install with:
+
+```sh
+gem install pgdexter
+```
+
+## How to Use
+
+Dexter needs a connection to your database and a log file to process.
+
+```sh
+dexter <database-url> <log-file>
+```
+
+This finds slow queries and generates output like:
+
+```
+SELECT * FROM ratings ORDER BY user_id LIMIT 10
+Starting cost: 3797.99
+Final cost: 0.5
+CREATE INDEX CONCURRENTLY ON ratings (user_id);
+```
+
+To be safe, Dexter does not create indexes unless you pass the `--create` flag.
+
+You can also pass a single statement with:
+
+```sh
+dexter <database-url> -s "SELECT * FROM ..."
+```
+
+## Options
+
+- `--min-time` - only consider queries that have consumed a certain amount of DB time (in minutes)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/dexter/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/dexter/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task default: :test

data/exe/dexter

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require "dexter"
+begin
+  Dexter::Client.new(ARGV).perform
+rescue Interrupt => e
+  # do nothing
+end

data/lib/dexter/log_parser.rb

@@ -0,0 +1,49 @@
+module Dexter
+  class LogParser
+    REGEX = /duration: (\d+\.\d+) ms  (statement|execute <unnamed>): (.+)/
+
+    def initialize(logfile, options = {})
+      @logfile = logfile
+      @min_time = options[:min_time] * 60000 # convert minutes to ms
+    end
+
+    def queries
+      @top_queries = {}
+
+      active_line = nil
+      duration = nil
+
+      File.foreach(@logfile) do |line|
+        if active_line
+          if line.include?(":  ")
+            process_entry(active_line, duration)
+            active_line = nil
+            duration = nil
+          else
+            active_line << line
+          end
+        end
+
+        if !active_line && m = REGEX.match(line.chomp)
+          duration = m[1].to_f
+          active_line = m[3]
+        else
+          # skip
+        end
+      end
+      process_entry(active_line, duration) if active_line
+
+      @top_queries.select { |_, v| v[:total_time] > @min_time }.map { |_, v| v[:query] }
+    end
+
+    private
+
+    def process_entry(query, duration)
+      return unless query =~ /SELECT/i
+      fingerprint = PgQuery.fingerprint(query)
+      @top_queries[fingerprint] ||= {calls: 0, total_time: 0, query: query}
+      @top_queries[fingerprint][:calls] += 1
+      @top_queries[fingerprint][:total_time] += duration
+    end
+  end
+end

data/lib/dexter/version.rb

@@ -0,0 +1,3 @@
+module Dexter
+  VERSION = "0.1.0"
+end

data/lib/dexter.rb

@@ -0,0 +1,265 @@
+require "dexter/version"
+require "slop"
+require "pg"
+require "pg_query"
+require "time"
+require "set"
+require "dexter/log_parser"
+
+module Dexter
+  class Client
+    attr_reader :arguments, :options
+
+    def initialize(args)
+      @arguments, @options = parse_args(args)
+    end
+
+    def perform
+      abort "Missing database url" if arguments.empty?
+      abort "Too many arguments" if arguments.size > 2
+
+      # get queries
+      queries = []
+      if options[:s]
+        queries << options[:s]
+      end
+      if arguments[1]
+        begin
+          parser = LogParser.new(arguments[1], min_time: options[:min_time])
+          queries.concat(parser.queries)
+        rescue Errno::ENOENT
+          abort "Log file not found"
+        end
+      end
+
+      # narrow down queries and tables
+      tables, queries = narrow_queries(queries)
+      return if tables.empty?
+
+      # get ready for hypothetical indexes
+      select_all("SET client_min_messages = warning")
+      select_all("CREATE EXTENSION IF NOT EXISTS hypopg")
+      select_all("SELECT hypopg_reset()")
+
+      # ensure tables have recently been analyzed
+      analyze_tables(tables)
+
+      # get initial plans
+      initial_plans = {}
+      queries.each do |query|
+        begin
+          initial_plans[query] = plan(query)
+        rescue PG::Error
+          # do nothing
+        end
+      end
+      queries.select! { |q| initial_plans[q] }
+
+      # get existing indexes
+      index_set = Set.new
+      indexes(tables).each do |index|
+        # TODO make sure btree
+        index_set << [index["table"], index["columns"]]
+      end
+
+      # create hypothetical indexes
+      candidates = {}
+      columns(tables).each do |col|
+        unless index_set.include?([col[:table], [col[:column]]])
+          candidates[col] = select_all("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{col[:table]} (#{[col[:column]].join(", ")})');").first["indexname"]
+        end
+      end
+
+      new_indexes = []
+      queries.each do |query|
+        starting_cost = initial_plans[query]["Total Cost"]
+        plan2 = plan(query)
+        cost2 = plan2["Total Cost"]
+        best_indexes = []
+
+        candidates.each do |col, index_name|
+          if plan2.inspect.include?(index_name)
+            best_indexes << {
+              table: col[:table],
+              columns: [col[:column]]
+            }
+          end
+        end
+
+        puts query
+        puts "Starting cost: #{starting_cost}"
+        puts "Final cost: #{cost2}"
+
+        # must make it 20% faster
+        if cost2 < starting_cost * 0.8
+          new_indexes.concat(best_indexes)
+          best_indexes.each do |index|
+            puts "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")});"
+          end
+        else
+          puts "Nope!"
+        end
+        puts
+      end
+
+      # create indexes
+      if new_indexes.any?
+        puts "Indexes to be created:"
+        new_indexes.uniq.sort_by(&:to_a).each do |index|
+          statement = "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")})"
+          puts "#{statement};"
+          select_all(statement) if options[:create]
+        end
+      end
+    end
+
+    def conn
+      @conn ||= begin
+        uri = URI.parse(arguments[0])
+        config = {
+          host: uri.host,
+          port: uri.port,
+          dbname: uri.path.sub(/\A\//, ""),
+          user: uri.user,
+          password: uri.password,
+          connect_timeout: 3
+        }.reject { |_, value| value.to_s.empty? }
+        PG::Connection.new(config)
+      end
+    rescue PG::ConnectionBad
+      abort "Bad database url"
+    end
+
+    def select_all(query)
+      conn.exec(query).to_a
+    end
+
+    def plan(query)
+      JSON.parse(select_all("EXPLAIN (FORMAT JSON) #{query}").first["QUERY PLAN"]).first["Plan"]
+    end
+
+    def narrow_queries(queries)
+      result = select_all <<-SQL
+        SELECT
+          table_name
+        FROM
+          information_schema.tables
+        WHERE
+          table_catalog = current_database() AND
+          table_schema NOT IN ('pg_catalog', 'information_schema')
+      SQL
+      possible_tables = Set.new(result.map { |r| r["table_name"] })
+
+      tables = queries.flat_map { |q| PgQuery.parse(q).tables }.uniq.select { |t| possible_tables.include?(t) }
+
+      [tables, queries.select { |q| PgQuery.parse(q).tables.all? { |t| possible_tables.include?(t) } }]
+    end
+
+    def columns(tables)
+      columns = select_all <<-SQL
+        SELECT
+          table_name,
+          column_name
+        FROM
+          information_schema.columns
+        WHERE
+          table_schema = 'public' AND
+          table_name IN (#{tables.map { |t| quote(t) }.join(", ")})
+      SQL
+
+      columns.map { |v| {table: v["table_name"], column: v["column_name"]} }
+    end
+
+    def indexes(tables)
+      select_all(<<-SQL
+        SELECT
+          schemaname AS schema,
+          t.relname AS table,
+          ix.relname AS name,
+          regexp_replace(pg_get_indexdef(i.indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
+          regexp_replace(pg_get_indexdef(i.indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
+          indisunique AS unique,
+          indisprimary AS primary,
+          indisvalid AS valid,
+          indexprs::text,
+          indpred::text,
+          pg_get_indexdef(i.indexrelid) AS definition
+        FROM
+          pg_index i
+        INNER JOIN
+          pg_class t ON t.oid = i.indrelid
+        INNER JOIN
+          pg_class ix ON ix.oid = i.indexrelid
+        LEFT JOIN
+          pg_stat_user_indexes ui ON ui.indexrelid = i.indexrelid
+        WHERE
+          t.relname IN (#{tables.map { |t| quote(t) }.join(", ")}) AND
+          schemaname IS NOT NULL AND
+          indisvalid = 't' AND
+          indexprs IS NULL AND
+          indpred IS NULL
+        ORDER BY
+          1, 2
+      SQL
+      ).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
+    end
+
+    def unquote(part)
+      if part && part.start_with?('"')
+        part[1..-2]
+      else
+        part
+      end
+    end
+
+    def analyze_tables(tables)
+      analyze_stats = select_all <<-SQL
+        SELECT
+          schemaname AS schema,
+          relname AS table,
+          last_analyze,
+          last_autoanalyze
+        FROM
+          pg_stat_user_tables
+        WHERE
+          relname IN (#{tables.map { |t| quote(t) }.join(", ")})
+      SQL
+
+      last_analyzed = {}
+      analyze_stats.each do |stats|
+        last_analyzed[stats["table"]] = Time.parse(stats["last_analyze"]) if stats["last_analyze"]
+      end
+
+      tables.each do |table|
+        if !last_analyzed[table] || last_analyzed[table] < Time.now - 3600
+          puts "Analyzing #{table}"
+          select_all("ANALYZE #{table}")
+        end
+      end
+    end
+
+    def quote(value)
+      if value.is_a?(String)
+        "'#{quote_string(value)}'"
+      else
+        value
+      end
+    end
+
+    # activerecord
+    def quote_string(s)
+      s.gsub(/\\/, '\&\&').gsub(/'/, "''")
+    end
+
+    def parse_args(args)
+      opts = Slop.parse(args) do |o|
+        o.boolean "--create", default: false
+        o.string "-s"
+        o.float "--min-time", default: 0
+      end
+      [opts.arguments, opts.to_hash]
+    rescue Slop::Error => e
+      abort e.message
+    end
+  end
+end

data/pgdexter.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "dexter/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "pgdexter"
+  spec.version       = Dexter::VERSION
+  spec.authors       = ["Andrew Kane"]
+  spec.email         = ["andrew@chartkick.com"]
+
+  spec.summary       = "An automatic indexer for Postgres"
+  spec.homepage      = "https://github.com/ankane/dexter"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "slop", ">= 4.2.0"
+  spec.add_dependency "pg"
+  spec.add_dependency "pg_query"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: pgdexter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-06-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- andrew@chartkick.com
+executables:
+- dexter
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/dexter
+- lib/dexter.rb
+- lib/dexter/log_parser.rb
+- lib/dexter/version.rb
+- pgdexter.gemspec
+homepage: https://github.com/ankane/dexter
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: An automatic indexer for Postgres
+test_files: []