pgdexter 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12871a558deb7cd0cb5067476415a34164d5d906
-  data.tar.gz: 6a293f4992bacb524a37bd7f7882851d364f6080
+  metadata.gz: a9dd13cbe8dc39e26b90bdc13abd16201ccec455
+  data.tar.gz: bc4693ab0595c9dc1e38b720c294b8e600d597c5
 SHA512:
-  metadata.gz: 846f1efba33eba3144e0918348068fc34fefe1ef8dfb62671231ccb01a836b2274b6ac6b43737484d43e0057556898b544aa12142df27860a881b4b9fa43b973
-  data.tar.gz: 676ad00ea2529681fbfe09e9982a30e49960a03f3dd25733ba206e2fed6bb0708789ec543acb193343b9ec0b31c0ecd12fd4f9d1adbbd9393324acd33322864c
+  metadata.gz: 5f96e60ac1660786dd0f059ec0e88a4f017c056f73eb0be07bb6d0d9ad55332a3c36c72cd9fb7f8b3a5ed370254b361f29979baa031850d93800ce8ad74775d2
+  data.tar.gz: f6b5a5a8487ea44eb38a4aca3ec59aaaea6c8cdd2c3e40267203fa29b4999d03a507fbe8bbe7e4207a5110c7bcc1e649dba624b4eb11e859be1f432e705095a8

data/README.md

@@ -33,28 +33,28 @@ gem install pgdexter
 Dexter needs a connection to your database and a log file to process.
 
 ```sh
-dexter <database-url> <log-file>
+tail -F -n +1 <log-file> | dexter <database-url>
 ```
 
 This finds slow queries and generates output like:
 
+```log
+2017-06-25T17:52:19+00:00 Started
+2017-06-25T17:52:22+00:00 Processing 189 new query fingerprints
+2017-06-25T17:52:22+00:00 Index found: genres_movies (genre_id)
+2017-06-25T17:52:22+00:00 Index found: genres_movies (movie_id)
+2017-06-25T17:52:22+00:00 Index found: movies (title)
+2017-06-25T17:52:22+00:00 Index found: ratings (movie_id)
+2017-06-25T17:52:22+00:00 Index found: ratings (rating)
+2017-06-25T17:52:22+00:00 Index found: ratings (user_id)
+2017-06-25T17:53:22+00:00 Processing 12 new query fingerprints
 ```
-SELECT * FROM ratings ORDER BY user_id LIMIT 10
-Starting cost: 3797.99
-Final cost: 0.5
-CREATE INDEX CONCURRENTLY ON ratings (user_id);
-```
-
-To be safe, Dexter does not create indexes unless you pass the `--create` flag.
-
-You can also pass a single statement with:
 
-```sh
-dexter <database-url> -s "SELECT * FROM ..."
-```
+To be safe, Dexter will not create indexes unless you pass the `--create` flag.
 
 ## Options
 
+- `--interval` - time to wait between processing queries
 - `--min-time` - only consider queries that have consumed a certain amount of DB time (in minutes)
 
 ## Contributing

data/lib/dexter/indexer.rb

@@ -0,0 +1,249 @@
+module Dexter
+  class Indexer
+    attr_reader :client
+
+    def initialize(client)
+      @client = client
+
+      select_all("SET client_min_messages = warning")
+      select_all("CREATE EXTENSION IF NOT EXISTS hypopg")
+    end
+
+    def process_queries(queries)
+      # narrow down queries and tables
+      tables, queries = narrow_queries(queries)
+      return [] if tables.empty?
+
+      # get ready for hypothetical indexes
+      select_all("SELECT hypopg_reset()")
+
+      # ensure tables have recently been analyzed
+      analyze_tables(tables)
+
+      # get initial plans
+      initial_plans = {}
+      queries.each do |query|
+        begin
+          initial_plans[query] = plan(query)
+        rescue PG::Error
+          # do nothing
+        end
+      end
+      queries.select! { |q| initial_plans[q] }
+
+      # get existing indexes
+      index_set = Set.new
+      indexes(tables).each do |index|
+        # TODO make sure btree
+        index_set << [index["table"], index["columns"]]
+      end
+
+      # create hypothetical indexes
+      candidates = {}
+      columns(tables).each do |col|
+        unless index_set.include?([col[:table], [col[:column]]])
+          candidates[col] = select_all("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{col[:table]} (#{[col[:column]].join(", ")})');").first["indexname"]
+        end
+      end
+
+      queries_by_index = {}
+
+      new_indexes = []
+      queries.each do |query|
+        starting_cost = initial_plans[query]["Total Cost"]
+        plan2 = plan(query)
+        cost2 = plan2["Total Cost"]
+        best_indexes = []
+
+        candidates.each do |col, index_name|
+          if plan2.inspect.include?(index_name) && cost2 < starting_cost * 0.5
+            best_indexes << {
+              table: col[:table],
+              columns: [col[:column]]
+            }
+            (queries_by_index[best_indexes.last] ||= []) << {
+              starting_cost: starting_cost,
+              final_cost: cost2,
+              query: query
+            }
+          end
+        end
+
+        new_indexes.concat(best_indexes)
+      end
+
+      new_indexes = new_indexes.uniq.sort_by(&:to_a)
+
+      # create indexes
+      if new_indexes.any?
+        new_indexes.each do |index|
+          index[:queries] = queries_by_index[index]
+
+          log "Index found: #{index[:table]} (#{index[:columns].join(", ")})"
+          # log "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")});"
+          # index[:queries].sort_by { |q| fingerprints[q[:query]] }.each do |query|
+          #   log "Query #{fingerprints[query[:query]]} (Cost: #{query[:starting_cost]} -> #{query[:final_cost]})"
+          #   puts
+          #   puts query[:query]
+          #   puts
+          # end
+        end
+
+        new_indexes.each do |index|
+          statement = "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")})"
+          # puts "#{statement};"
+          if client.options[:create]
+            log "Creating index: #{statement}"
+            started_at = Time.now
+            select_all(statement)
+            log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
+          end
+        end
+      end
+
+      new_indexes
+    end
+
+    def conn
+      @conn ||= begin
+        uri = URI.parse(client.arguments[0])
+        config = {
+          host: uri.host,
+          port: uri.port,
+          dbname: uri.path.sub(/\A\//, ""),
+          user: uri.user,
+          password: uri.password,
+          connect_timeout: 3
+        }.reject { |_, value| value.to_s.empty? }
+        PG::Connection.new(config)
+      end
+    rescue PG::ConnectionBad
+      abort "Bad database url"
+    end
+
+    def select_all(query)
+      conn.exec(query).to_a
+    end
+
+    def plan(query)
+      JSON.parse(select_all("EXPLAIN (FORMAT JSON) #{query}").first["QUERY PLAN"]).first["Plan"]
+    end
+
+    def narrow_queries(queries)
+      result = select_all <<-SQL
+        SELECT
+          table_name
+        FROM
+          information_schema.tables
+        WHERE
+          table_catalog = current_database() AND
+          table_schema NOT IN ('pg_catalog', 'information_schema')
+      SQL
+      possible_tables = Set.new(result.map { |r| r["table_name"] })
+
+      tables = queries.flat_map { |q| PgQuery.parse(q).tables }.uniq.select { |t| possible_tables.include?(t) }
+
+      [tables, queries.select { |q| PgQuery.parse(q).tables.all? { |t| possible_tables.include?(t) } }]
+    end
+
+    def columns(tables)
+      columns = select_all <<-SQL
+        SELECT
+          table_name,
+          column_name
+        FROM
+          information_schema.columns
+        WHERE
+          table_schema = 'public' AND
+          table_name IN (#{tables.map { |t| quote(t) }.join(", ")})
+      SQL
+
+      columns.map { |v| {table: v["table_name"], column: v["column_name"]} }
+    end
+
+    def indexes(tables)
+      select_all(<<-SQL
+        SELECT
+          schemaname AS schema,
+          t.relname AS table,
+          ix.relname AS name,
+          regexp_replace(pg_get_indexdef(i.indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
+          regexp_replace(pg_get_indexdef(i.indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
+          indisunique AS unique,
+          indisprimary AS primary,
+          indisvalid AS valid,
+          indexprs::text,
+          indpred::text,
+          pg_get_indexdef(i.indexrelid) AS definition
+        FROM
+          pg_index i
+        INNER JOIN
+          pg_class t ON t.oid = i.indrelid
+        INNER JOIN
+          pg_class ix ON ix.oid = i.indexrelid
+        LEFT JOIN
+          pg_stat_user_indexes ui ON ui.indexrelid = i.indexrelid
+        WHERE
+          t.relname IN (#{tables.map { |t| quote(t) }.join(", ")}) AND
+          schemaname IS NOT NULL AND
+          indisvalid = 't' AND
+          indexprs IS NULL AND
+          indpred IS NULL
+        ORDER BY
+          1, 2
+      SQL
+      ).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
+    end
+
+    def unquote(part)
+      if part && part.start_with?('"')
+        part[1..-2]
+      else
+        part
+      end
+    end
+
+    def analyze_tables(tables)
+      analyze_stats = select_all <<-SQL
+        SELECT
+          schemaname AS schema,
+          relname AS table,
+          last_analyze,
+          last_autoanalyze
+        FROM
+          pg_stat_user_tables
+        WHERE
+          relname IN (#{tables.map { |t| quote(t) }.join(", ")})
+      SQL
+
+      last_analyzed = {}
+      analyze_stats.each do |stats|
+        last_analyzed[stats["table"]] = Time.parse(stats["last_analyze"]) if stats["last_analyze"]
+      end
+
+      tables.each do |table|
+        if !last_analyzed[table] || last_analyzed[table] < Time.now - 3600
+          log "Analyzing #{table}"
+          select_all("ANALYZE #{table}")
+        end
+      end
+    end
+
+    def quote(value)
+      if value.is_a?(String)
+        "'#{quote_string(value)}'"
+      else
+        value
+      end
+    end
+
+    # activerecord
+    def quote_string(s)
+      s.gsub(/\\/, '\&\&').gsub(/'/, "''")
+    end
+
+    def log(message)
+      puts "#{Time.now.iso8601} #{message}"
+    end
+  end
+end

data/lib/dexter/log_parser.rb

@@ -2,18 +2,38 @@ module Dexter
   class LogParser
     REGEX = /duration: (\d+\.\d+) ms  (statement|execute <unnamed>): (.+)/
 
-    def initialize(logfile, options = {})
+    def initialize(logfile, client)
       @logfile = logfile
-      @min_time = options[:min_time] * 60000 # convert minutes to ms
-    end
-
-    def queries
+      @min_time = client.options[:min_time] * 60000 # convert minutes to ms
       @top_queries = {}
+      @indexer = Indexer.new(client)
+      @new_queries = Set.new
+      @new_queries_mutex = Mutex.new
+      @process_queries_mutex = Mutex.new
+      @last_checked_at = {}
+
+      log "Started"
+
+      if @logfile == STDIN
+        Thread.abort_on_exception = true
 
+        @timer_thread = Thread.new do
+          sleep(3) # starting sleep
+          loop do
+            @process_queries_mutex.synchronize do
+              process_queries
+            end
+            sleep(client.options[:interval])
+          end
+        end
+      end
+    end
+
+    def perform
       active_line = nil
       duration = nil
 
-      File.foreach(@logfile) do |line|
+      each_line do |line|
         if active_line
           if line.include?(":  ")
             process_entry(active_line, duration)
@@ -33,17 +53,72 @@ module Dexter
       end
       process_entry(active_line, duration) if active_line
 
-      @top_queries.select { |_, v| v[:total_time] > @min_time }.map { |_, v| v[:query] }
+      @process_queries_mutex.synchronize do
+        process_queries
+      end
     end
 
     private
 
+    def each_line
+      if @logfile == STDIN
+        STDIN.each_line do |line|
+          yield line
+        end
+      else
+        File.foreach(@logfile) do |line|
+          yield line
+        end
+      end
+    end
+
     def process_entry(query, duration)
       return unless query =~ /SELECT/i
-      fingerprint = PgQuery.fingerprint(query)
-      @top_queries[fingerprint] ||= {calls: 0, total_time: 0, query: query}
+      fingerprint =
+        begin
+          PgQuery.fingerprint(query)
+        rescue PgQuery::ParseError
+          # do nothing
+        end
+      return unless fingerprint
+
+      @top_queries[fingerprint] ||= {calls: 0, total_time: 0}
       @top_queries[fingerprint][:calls] += 1
       @top_queries[fingerprint][:total_time] += duration
+      @top_queries[fingerprint][:query] = query
+      @new_queries_mutex.synchronize do
+        @new_queries << fingerprint
+      end
+    end
+
+    def process_queries
+      new_queries = nil
+
+      @new_queries_mutex.synchronize do
+        new_queries = @new_queries.dup
+        @new_queries.clear
+      end
+
+      now = Time.now
+      min_checked_at = now - 3600 # don't recheck for an hour
+      queries = []
+      fingerprints = {}
+      @top_queries.each do |k, v|
+        if new_queries.include?(k) && v[:total_time] > @min_time && (!@last_checked_at[k] || @last_checked_at[k] < min_checked_at)
+          fingerprints[v[:query]] = k
+          queries << v[:query]
+          @last_checked_at[k] = now
+        end
+      end
+
+      log "Processing #{queries.size} new query fingerprints"
+      if queries.any?
+        @indexer.process_queries(queries)
+      end
+    end
+
+    def log(message)
+      puts "#{Time.now.iso8601} #{message}"
     end
   end
 end

data/lib/dexter/version.rb

@@ -1,3 +1,3 @@
 module Dexter
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/dexter.rb

@@ -4,6 +4,8 @@ require "pg"
 require "pg_query"
 require "time"
 require "set"
+require "thread"
+require "dexter/indexer"
 require "dexter/log_parser"
 
 module Dexter
@@ -22,240 +24,26 @@ module Dexter
       queries = []
       if options[:s]
         queries << options[:s]
+        Indexer.new(self).process_queries(queries)
       end
       if arguments[1]
         begin
-          parser = LogParser.new(arguments[1], min_time: options[:min_time])
-          queries.concat(parser.queries)
+          LogParser.new(arguments[1], self).perform
         rescue Errno::ENOENT
           abort "Log file not found"
         end
       end
-
-      # narrow down queries and tables
-      tables, queries = narrow_queries(queries)
-      return if tables.empty?
-
-      # get ready for hypothetical indexes
-      select_all("SET client_min_messages = warning")
-      select_all("CREATE EXTENSION IF NOT EXISTS hypopg")
-      select_all("SELECT hypopg_reset()")
-
-      # ensure tables have recently been analyzed
-      analyze_tables(tables)
-
-      # get initial plans
-      initial_plans = {}
-      queries.each do |query|
-        begin
-          initial_plans[query] = plan(query)
-        rescue PG::Error
-          # do nothing
-        end
-      end
-      queries.select! { |q| initial_plans[q] }
-
-      # get existing indexes
-      index_set = Set.new
-      indexes(tables).each do |index|
-        # TODO make sure btree
-        index_set << [index["table"], index["columns"]]
-      end
-
-      # create hypothetical indexes
-      candidates = {}
-      columns(tables).each do |col|
-        unless index_set.include?([col[:table], [col[:column]]])
-          candidates[col] = select_all("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{col[:table]} (#{[col[:column]].join(", ")})');").first["indexname"]
-        end
-      end
-
-      new_indexes = []
-      queries.each do |query|
-        starting_cost = initial_plans[query]["Total Cost"]
-        plan2 = plan(query)
-        cost2 = plan2["Total Cost"]
-        best_indexes = []
-
-        candidates.each do |col, index_name|
-          if plan2.inspect.include?(index_name)
-            best_indexes << {
-              table: col[:table],
-              columns: [col[:column]]
-            }
-          end
-        end
-
-        puts query
-        puts "Starting cost: #{starting_cost}"
-        puts "Final cost: #{cost2}"
-
-        # must make it 20% faster
-        if cost2 < starting_cost * 0.8
-          new_indexes.concat(best_indexes)
-          best_indexes.each do |index|
-            puts "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")});"
-          end
-        else
-          puts "Nope!"
-        end
-        puts
-      end
-
-      # create indexes
-      if new_indexes.any?
-        puts "Indexes to be created:"
-        new_indexes.uniq.sort_by(&:to_a).each do |index|
-          statement = "CREATE INDEX CONCURRENTLY ON #{index[:table]} (#{index[:columns].join(", ")})"
-          puts "#{statement};"
-          select_all(statement) if options[:create]
-        end
-      end
-    end
-
-    def conn
-      @conn ||= begin
-        uri = URI.parse(arguments[0])
-        config = {
-          host: uri.host,
-          port: uri.port,
-          dbname: uri.path.sub(/\A\//, ""),
-          user: uri.user,
-          password: uri.password,
-          connect_timeout: 3
-        }.reject { |_, value| value.to_s.empty? }
-        PG::Connection.new(config)
-      end
-    rescue PG::ConnectionBad
-      abort "Bad database url"
-    end
-
-    def select_all(query)
-      conn.exec(query).to_a
-    end
-
-    def plan(query)
-      JSON.parse(select_all("EXPLAIN (FORMAT JSON) #{query}").first["QUERY PLAN"]).first["Plan"]
-    end
-
-    def narrow_queries(queries)
-      result = select_all <<-SQL
-        SELECT
-          table_name
-        FROM
-          information_schema.tables
-        WHERE
-          table_catalog = current_database() AND
-          table_schema NOT IN ('pg_catalog', 'information_schema')
-      SQL
-      possible_tables = Set.new(result.map { |r| r["table_name"] })
-
-      tables = queries.flat_map { |q| PgQuery.parse(q).tables }.uniq.select { |t| possible_tables.include?(t) }
-
-      [tables, queries.select { |q| PgQuery.parse(q).tables.all? { |t| possible_tables.include?(t) } }]
-    end
-
-    def columns(tables)
-      columns = select_all <<-SQL
-        SELECT
-          table_name,
-          column_name
-        FROM
-          information_schema.columns
-        WHERE
-          table_schema = 'public' AND
-          table_name IN (#{tables.map { |t| quote(t) }.join(", ")})
-      SQL
-
-      columns.map { |v| {table: v["table_name"], column: v["column_name"]} }
-    end
-
-    def indexes(tables)
-      select_all(<<-SQL
-        SELECT
-          schemaname AS schema,
-          t.relname AS table,
-          ix.relname AS name,
-          regexp_replace(pg_get_indexdef(i.indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
-          regexp_replace(pg_get_indexdef(i.indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
-          indisunique AS unique,
-          indisprimary AS primary,
-          indisvalid AS valid,
-          indexprs::text,
-          indpred::text,
-          pg_get_indexdef(i.indexrelid) AS definition
-        FROM
-          pg_index i
-        INNER JOIN
-          pg_class t ON t.oid = i.indrelid
-        INNER JOIN
-          pg_class ix ON ix.oid = i.indexrelid
-        LEFT JOIN
-          pg_stat_user_indexes ui ON ui.indexrelid = i.indexrelid
-        WHERE
-          t.relname IN (#{tables.map { |t| quote(t) }.join(", ")}) AND
-          schemaname IS NOT NULL AND
-          indisvalid = 't' AND
-          indexprs IS NULL AND
-          indpred IS NULL
-        ORDER BY
-          1, 2
-      SQL
-      ).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
-    end
-
-    def unquote(part)
-      if part && part.start_with?('"')
-        part[1..-2]
-      else
-        part
+      if !options[:s] && !arguments[1]
+        LogParser.new(STDIN, self).perform
       end
     end
 
-    def analyze_tables(tables)
-      analyze_stats = select_all <<-SQL
-        SELECT
-          schemaname AS schema,
-          relname AS table,
-          last_analyze,
-          last_autoanalyze
-        FROM
-          pg_stat_user_tables
-        WHERE
-          relname IN (#{tables.map { |t| quote(t) }.join(", ")})
-      SQL
-
-      last_analyzed = {}
-      analyze_stats.each do |stats|
-        last_analyzed[stats["table"]] = Time.parse(stats["last_analyze"]) if stats["last_analyze"]
-      end
-
-      tables.each do |table|
-        if !last_analyzed[table] || last_analyzed[table] < Time.now - 3600
-          puts "Analyzing #{table}"
-          select_all("ANALYZE #{table}")
-        end
-      end
-    end
-
-    def quote(value)
-      if value.is_a?(String)
-        "'#{quote_string(value)}'"
-      else
-        value
-      end
-    end
-
-    # activerecord
-    def quote_string(s)
-      s.gsub(/\\/, '\&\&').gsub(/'/, "''")
-    end
-
     def parse_args(args)
       opts = Slop.parse(args) do |o|
         o.boolean "--create", default: false
         o.string "-s"
         o.float "--min-time", default: 0
+        o.integer "--interval", default: 60
       end
       [opts.arguments, opts.to_hash]
     rescue Slop::Error => e

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgdexter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-24 00:00:00.000000000 Z
+date: 2017-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -95,6 +95,7 @@ files:
 - Rakefile
 - exe/dexter
 - lib/dexter.rb
+- lib/dexter/indexer.rb
 - lib/dexter/log_parser.rb
 - lib/dexter/version.rb
 - pgdexter.gemspec