pghero 1.1.4 → 1.2.0

data/app/views/pg_hero/home/system.html.erb

@@ -4,6 +4,9 @@
   <h1>CPU</h1>
   <div style="margin-bottom: 20px;"><%= line_chart cpu_usage_path, max: 100, colors: ["#5bc0de"], library: {pointSize: 0, lineWidth: 5} %></div>
 
+  <h1>Load</h1>
+  <div style="margin-bottom: 20px;"><%= line_chart load_stats_path, colors: ["#5bc0de", "#d9534f"], library: {pointSize: 0, lineWidth: 5} %></div>
+
   <h1>Connections</h1>
   <div style="margin-bottom: 20px;"><%= line_chart connection_stats_path, colors: ["#5bc0de"], library: {pointSize: 0, lineWidth: 5} %></div>
 

data/config/routes.rb

@@ -8,9 +8,11 @@ PgHero::Engine.routes.draw do
     get "cpu_usage", to: "home#cpu_usage"
     get "connection_stats", to: "home#connection_stats"
     get "replication_lag_stats", to: "home#replication_lag_stats"
+    get "load_stats", to: "home#load_stats"
     get "explain", to: "home#explain"
     get "tune", to: "home#tune"
     get "connections", to: "home#connections"
+    get "maintenance", to: "home#maintenance"
     post "kill", to: "home#kill"
     post "kill_long_running_queries", to: "home#kill_long_running_queries"
     post "kill_all", to: "home#kill_all"

data/guides/Rails.md

@@ -18,6 +18,16 @@ mount PgHero::Engine, at: "pghero"
 
 Be sure to [secure the dashboard](#security) in production.
 
+### Suggested Indexes
+
+PgHero can suggest indexes to add. To enable, add to your Gemfile:
+
+```ruby
+gem 'pg_query'
+```
+
+and make sure [query stats](#query-stats) are enabled. Read about how it works [here](Suggested-Indexes.md).
+
 ## Insights
 
 ```ruby
@@ -55,6 +65,13 @@ PgHero.query_stats
 PgHero.slow_queries
 ```
 
+Suggested indexes
+
+```ruby
+PgHero.suggested_indexes
+PgHero.best_index(query)
+```
+
 Security
 
 ```ruby

data/guides/Suggested-Indexes.md

@@ -0,0 +1,19 @@
+# How PgHero Suggests Indexes
+
+1. Get the most time-consuming queries from [pg_stat_statements](http://www.postgresql.org/docs/9.3/static/pgstatstatements.html).
+
+2. Parse queries with [pg_query](https://github.com/lfittl/pg_query).  Look for a single table with a `WHERE` clause that consists of only `=`, `IN`, `IS NULL` or `IS NOT NULL` and/or an `ORDER BY` clause.
+
+3. Use the [pg_stats](http://www.postgresql.org/docs/current/static/view-pg-stats.html) view to get estimates about distinct rows and percent of `NULL` values for each column.
+
+4. For each column in the `WHERE` clause, sort by the highest [cardinality](https://en.wikipedia.org/wiki/Cardinality_(SQL_statements)) (most unique values). This allows the database to narrow its search the fastest. Perform [row estimation](http://www.postgresql.org/docs/current/static/row-estimation-examples.html) to get the expected number of rows as we add columns to the index.
+
+5. Continue this process with columns in the `ORDER BY` clause.
+
+6. To make sure we don’t add useless columns, stop once we narrow it down to 50 rows in steps 5 or 6. Also, recheck the last columns to make sure they add value.
+
+7. Profit :moneybag:
+
+## TODO
+
+- examples

data/lib/pghero.rb

@@ -17,15 +17,25 @@ module PgHero
   end
 
   class << self
-    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :total_connections_threshold, :env
+    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :total_connections_threshold, :cache_hit_rate_threshold, :env, :show_migrations
   end
   self.long_running_query_sec = (ENV["PGHERO_LONG_RUNNING_QUERY_SEC"] || 60).to_i
   self.slow_query_ms = (ENV["PGHERO_SLOW_QUERY_MS"] || 20).to_i
   self.slow_query_calls = (ENV["PGHERO_SLOW_QUERY_CALLS"] || 100).to_i
   self.total_connections_threshold = (ENV["PGHERO_TOTAL_CONNECTIONS_THRESHOLD"] || 100).to_i
+  self.cache_hit_rate_threshold = 99
   self.env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+  self.show_migrations = true
 
   class << self
+    def time_zone=(time_zone)
+      @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
+    end
+
+    def time_zone
+      @time_zone || Time.zone
+    end
+
     def config
       Thread.current[:pghero_config] ||= begin
         path = "config/pghero.yml"
@@ -167,6 +177,39 @@ module PgHero
       ).first["rate"].to_f
     end
 
+    def table_caching
+      select_all <<-SQL
+        SELECT
+          relname AS table,
+          CASE WHEN heap_blks_hit + heap_blks_read = 0 THEN
+            0
+          ELSE
+            ROUND(1.0 * heap_blks_hit / (heap_blks_hit + heap_blks_read), 2)
+          END AS hit_rate
+        FROM
+          pg_statio_user_tables
+        ORDER BY
+          2 DESC, 1
+      SQL
+    end
+
+    def index_caching
+      select_all <<-SQL
+        SELECT
+          indexrelname AS index,
+          relname AS table,
+          CASE WHEN idx_blks_hit + idx_blks_read = 0 THEN
+            0
+          ELSE
+            ROUND(1.0 * idx_blks_hit / (idx_blks_hit + idx_blks_read), 2)
+          END AS hit_rate
+        FROM
+          pg_statio_user_indexes
+        ORDER BY
+          3 DESC, 1
+      SQL
+    end
+
     def index_usage
       select_all <<-SQL
         SELECT
@@ -288,24 +331,44 @@ module PgHero
       select_all("SELECT COUNT(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid()").first["count"].to_i
     end
 
-    def connection_sources
-      select_all <<-SQL
-        SELECT
-          application_name AS source,
-          client_addr AS ip,
-          COUNT(*) AS total_connections
-        FROM
-          pg_stat_activity
-        WHERE
-          pid <> pg_backend_pid()
-        GROUP BY
-          application_name,
-          ip
-        ORDER BY
-          COUNT(*) DESC,
-          application_name ASC,
-          client_addr ASC
-      SQL
+    def connection_sources(options = {})
+      if options[:by_database]
+        select_all <<-SQL
+          SELECT
+            application_name AS source,
+            client_addr AS ip,
+            datname AS database,
+            COUNT(*) AS total_connections
+          FROM
+            pg_stat_activity
+          WHERE
+            pid <> pg_backend_pid()
+          GROUP BY
+            1, 2, 3
+          ORDER BY
+            COUNT(*) DESC,
+            application_name ASC,
+            client_addr ASC
+        SQL
+      else
+        select_all <<-SQL
+          SELECT
+            application_name AS source,
+            client_addr AS ip,
+            COUNT(*) AS total_connections
+          FROM
+            pg_stat_activity
+          WHERE
+            pid <> pg_backend_pid()
+          GROUP BY
+            application_name,
+            ip
+          ORDER BY
+            COUNT(*) DESC,
+            application_name ASC,
+            client_addr ASC
+        SQL
+      end
     end
 
     # http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND
@@ -332,7 +395,7 @@ module PgHero
     def autovacuum_danger
       select_all <<-SQL
         SELECT
-          c.oid::regclass as table,
+          c.oid::regclass::text as table,
           (SELECT setting FROM pg_settings WHERE name = 'autovacuum_freeze_max_age')::int -
           GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) AS transactions_before_autovacuum
         FROM
@@ -347,6 +410,21 @@ module PgHero
       SQL
     end
 
+    def maintenance_info
+      select_all <<-SQL
+        SELECT
+          relname AS table,
+          last_vacuum,
+          last_autovacuum,
+          last_analyze,
+          last_autoanalyze
+        FROM
+          pg_stat_user_tables
+        ORDER BY
+          relname ASC
+      SQL
+    end
+
     def kill(pid)
       execute("SELECT pg_terminate_backend(#{pid.to_i})").first["pg_terminate_backend"] == "t"
     end
@@ -387,11 +465,19 @@ module PgHero
         query_stats << value
       end
       sort = options[:sort] || "total_minutes"
-      query_stats.sort_by { |q| -q[sort] }.first(100)
+      query_stats = query_stats.sort_by { |q| -q[sort] }.first(100)
+      if options[:min_average_time]
+        query_stats.reject! { |q| q["average_time"].to_f < options[:min_average_time] }
+      end
+      if options[:min_calls]
+        query_stats.reject! { |q| q["calls"].to_i < options[:min_calls] }
+      end
+      query_stats
     end
 
     def slow_queries(options = {})
-      query_stats(options).select { |q| q["calls"].to_i >= slow_query_calls.to_i && q["average_time"].to_i >= slow_query_ms.to_i }
+      query_stats = options[:query_stats] || self.query_stats(options.except(:query_stats))
+      query_stats.select { |q| q["calls"].to_i >= slow_query_calls.to_i && q["average_time"].to_i >= slow_query_ms.to_i }
     end
 
     def query_stats_available?
@@ -495,6 +581,14 @@ module PgHero
       rds_stats("ReplicaLag")
     end
 
+    def read_iops_stats
+      rds_stats("ReadIOPS")
+    end
+
+    def write_iops_stats
+      rds_stats("WriteIOPS")
+    end
+
     def rds_stats(metric_name)
       if system_stats_enabled?
         client =
@@ -655,8 +749,337 @@ module PgHero
       select_all("SELECT EXTRACT(EPOCH FROM NOW() - pg_last_xact_replay_timestamp()) AS replication_lag").first["replication_lag"].to_f
     end
 
+    # TODO parse array properly
+    # http://stackoverflow.com/questions/2204058/list-columns-with-indexes-in-postgresql
+    def indexes
+      select_all( <<-SQL
+        SELECT
+          t.relname AS table,
+          ix.relname AS name,
+          regexp_replace(pg_get_indexdef(indexrelid), '.*\\((.*)\\)', '\\1') AS columns,
+          regexp_replace(pg_get_indexdef(indexrelid), '.* USING (.*) \\(.*', '\\1') AS using,
+          indisunique AS unique,
+          indisprimary AS primary,
+          indisvalid AS valid,
+          indexprs::text,
+          indpred::text
+        FROM
+          pg_index i
+        INNER JOIN
+          pg_class t ON t.oid = i.indrelid
+        INNER JOIN
+          pg_class ix ON ix.oid = i.indexrelid
+        ORDER BY
+          1, 2
+      SQL
+      ).map { |v| v["columns"] = v["columns"].split(", "); v }
+    end
+
+    def duplicate_indexes
+      indexes = []
+
+      indexes_by_table = self.indexes.group_by { |i| i["table"] }
+      indexes_by_table.values.flatten.select { |i| i["primary"] == "f" && i["unique"] == "f" && !i["indexprs"] && !i["indpred"] && i["valid"] == "t" }.each do |index|
+        covering_index = indexes_by_table[index["table"]].find { |i| index_covers?(i["columns"], index["columns"]) && i["using"] == index["using"] && i["name"] != index["name"] && i["valid"] == "t" }
+        if covering_index
+          indexes << {"unneeded_index" => index, "covering_index" => covering_index}
+        end
+      end
+
+      indexes.sort_by { |i| ui = i["unneeded_index"]; [ui["table"], ui["columns"]] }
+    end
+
+    def suggested_indexes_enabled?
+      defined?(PgQuery) && query_stats_enabled?
+    end
+
+    # TODO clean this mess
+    def suggested_indexes_by_query(options = {})
+      best_indexes = {}
+
+      if suggested_indexes_enabled?
+        # get most time-consuming queries
+        queries = options[:queries] || (options[:query_stats] || self.query_stats(historical: true, start_at: 24.hours.ago)).map { |qs| qs["query"] }
+
+        # get best indexes for queries
+        best_indexes = best_index_helper(queries)
+
+        if best_indexes.any?
+          existing_columns = Hash.new { |hash, key| hash[key] = [] }
+          self.indexes.each do |i|
+            existing_columns[i["table"]] << i["columns"]
+          end
+
+          best_indexes.each do |query, best_index|
+            if best_index[:found]
+              index = best_index[:index]
+              covering_index = existing_columns[index[:table]].find { |e| index_covers?(e, index[:columns]) }
+              if covering_index
+                best_index[:covering_index] = covering_index
+                best_index[:explanation] = "Covered by index on (#{covering_index.join(", ")})"
+              end
+            end
+          end
+        end
+      end
+
+      best_indexes
+    end
+
+    def suggested_indexes(options = {})
+      indexes = []
+
+      (options[:suggested_indexes_by_query] || suggested_indexes_by_query(options)).select { |s, i| i[:found] && !i[:covering_index] }.group_by { |s, i| i[:index] }.each do |index, group|
+        indexes << index.merge(queries: group.map(&:first))
+      end
+
+      indexes.sort_by { |i| [i[:table], i[:columns]] }
+    end
+
+    def autoindex(options = {})
+      suggested_indexes.each do |index|
+        p index
+        if options[:create]
+          connection.execute("CREATE INDEX CONCURRENTLY ON #{quote_table_name(index[:table])} (#{index[:columns].map { |c| quote_table_name(c) }.join(",")})")
+        end
+      end
+    end
+
+    def autoindex_all(options = {})
+      config["databases"].keys.each do |database|
+        with(database) do
+          puts "Autoindexing #{database}..."
+          autoindex(options)
+        end
+      end
+    end
+
+    def best_index(statement, options = {})
+      best_index_helper([statement])[statement]
+    end
+
+    def column_stats(options = {})
+      tables = options[:table] ? Array(options[:table]) : nil
+      select_all <<-SQL
+        SELECT
+          tablename AS table,
+          attname AS column,
+          null_frac,
+          n_distinct,
+          n_live_tup
+        FROM
+          pg_stats
+        INNER JOIN
+          pg_class ON pg_class.relname = pg_stats.tablename
+        INNER JOIN
+          pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname
+        WHERE
+          #{tables ? "pg_class.relname IN (#{tables.map { |t| quote(t) }.join(", ")})" : "1 = 1"}
+        ORDER BY
+          1, 2
+      SQL
+    end
+
     private
 
+    def best_index_helper(statements)
+      indexes = {}
+
+      # see if this is a query we understand and can use
+      parts = {}
+      statements.each do |statement|
+        parts[statement] = best_index_structure(statement)
+      end
+
+      # get stats about columns for relevant tables
+      tables = parts.values.map { |t| t[:table] }.uniq
+      if tables.any?
+        column_stats = self.column_stats(table: tables).group_by { |i| i["table"] }
+      end
+
+      # find best index based on query structure and column stats
+      parts.each do |statement, structure|
+        index = {found: false}
+
+        if structure[:error]
+          index[:explanation] = structure[:error]
+        elsif structure[:table].start_with?("pg_")
+          index[:explanation] = "System table"
+        else
+          index[:structure] = structure
+
+          table = structure[:table]
+          where = structure[:where]
+          sort = structure[:sort]
+
+          ranks = Hash[column_stats[table].to_a.map { |r| [r["column"], r] }]
+
+          columns = (where + sort).map { |c| c[:column] }.uniq
+
+          if columns.any? && columns.all? { |c| ranks[c] }
+            first_desc = sort.index { |c| c[:direction] == "desc" }
+            if first_desc
+              sort = sort.first(first_desc + 1)
+            end
+            where = where.sort_by { |c| [row_estimates(ranks[c[:column]], nil, c[:op]), c[:column]] } + sort
+
+            index[:row_estimates] = Hash[where.map { |c| [c[:column], row_estimates(ranks[c[:column]], nil, c[:op]).round] }]
+
+            rows_left = ranks[where.first[:column]]["n_live_tup"].to_i
+            index[:rows] = rows_left
+
+            # no index needed if less than 500 rows
+            if rows_left >= 500
+
+              # if most values are unique, no need to index others
+              final_where = []
+              prev_rows_left = [rows_left]
+              where.each do |c|
+                next if final_where.include?(c[:column])
+                final_where << c[:column]
+                rows_left = row_estimates(ranks[c[:column]], rows_left, c[:op])
+                prev_rows_left << rows_left
+                if rows_left < 50
+                  break
+                end
+              end
+
+              index[:row_progression] = prev_rows_left.map(&:round)
+
+              # if the last indexes don't give us much, don't include
+              if prev_rows_left.last > 50
+                prev_rows_left.reverse!
+                (prev_rows_left.size - 1).times do |i|
+                  if prev_rows_left[i] > prev_rows_left[i + 1] * 0.1
+                    final_where.pop
+                  else
+                    break
+                  end
+                end
+              end
+
+              if final_where.any?
+                index[:found] = true
+                index[:index] = {table: table, columns: final_where}
+              end
+            else
+              index[:explanation] = "No index needed if less than 500 rows"
+            end
+          else
+            index[:explanation] = "No columns to index"
+          end
+        end
+
+        indexes[statement] = index
+      end
+
+      indexes
+    end
+
+    def best_index_structure(statement)
+      begin
+        tree = PgQuery.parse(statement).parsetree
+      rescue PgQuery::ParseError
+        return {error: "Parse error"}
+      end
+      return {error: "Unknown structure"} unless tree.size == 1
+
+      tree = tree.first
+      table = parse_table(tree) rescue nil
+      unless table
+        error =
+          case tree.keys.first
+          when "INSERT INTO"
+            "INSERT statement"
+          when "SET"
+            "SET statement"
+          else
+            "Unknown structure"
+          end
+        return {error: error}
+      end
+
+      select = tree["SELECT"] || tree["DELETE FROM"] || tree["UPDATE"]
+      where = (select["whereClause"] ? parse_where(select["whereClause"]) : []) rescue nil
+      return {error: "Unknown structure"} unless where
+
+      sort = (select["sortClause"] ? parse_sort(select["sortClause"]) : []) rescue nil
+      return {error: "Unknown structure"} unless sort
+
+      {table: table, where: where, sort: sort}
+    end
+
+    def index_covers?(indexed_columns, columns)
+      indexed_columns.first(columns.size) == columns
+    end
+
+    # TODO better row estimation
+    # http://www.postgresql.org/docs/current/static/row-estimation-examples.html
+    def row_estimates(stats, rows_left = nil, op = nil)
+      rows_left ||= stats["n_live_tup"].to_i
+      case op
+      when "null"
+        rows_left * stats["null_frac"].to_f
+      when "not_null"
+        rows_left * (1 - stats["null_frac"].to_f)
+      else
+        rows_left *= (1 - stats["null_frac"].to_f)
+        if stats["n_distinct"].to_f == 0
+          0
+        elsif stats["n_distinct"].to_f < 0
+          if stats["n_live_tup"].to_i > 0
+            (-1 / stats["n_distinct"].to_f) * (rows_left / stats["n_live_tup"].to_f)
+          else
+            0
+          end
+        else
+          rows_left / stats["n_distinct"].to_f
+        end
+      end
+    end
+
+    def parse_table(tree)
+      case tree.keys.first
+      when "SELECT"
+        tree["SELECT"]["fromClause"].first["RANGEVAR"]["relname"]
+      when "DELETE FROM"
+        tree["DELETE FROM"]["relation"]["RANGEVAR"]["relname"]
+      when "UPDATE"
+        tree["UPDATE"]["relation"]["RANGEVAR"]["relname"]
+      else
+        nil
+      end
+    end
+
+    # TODO capture values
+    def parse_where(tree)
+      if tree["AEXPR AND"]
+        left = parse_where(tree["AEXPR AND"]["lexpr"])
+        right = parse_where(tree["AEXPR AND"]["rexpr"])
+        if left && right
+          left + right
+        end
+      elsif tree["AEXPR"] && ["="].include?(tree["AEXPR"]["name"].first)
+        [{column: tree["AEXPR"]["lexpr"]["COLUMNREF"]["fields"].last, op: tree["AEXPR"]["name"].first}]
+      elsif tree["AEXPR IN"] && tree["AEXPR IN"]["name"].first == "="
+        [{column: tree["AEXPR IN"]["lexpr"]["COLUMNREF"]["fields"].last, op: "in"}]
+      elsif tree["NULLTEST"]
+        op = tree["NULLTEST"]["nulltesttype"] == 1 ? "not_null" : "null"
+        [{column: tree["NULLTEST"]["arg"]["COLUMNREF"]["fields"].last, op: op}]
+      else
+        nil
+      end
+    end
+
+    def parse_sort(sort_clause)
+      sort_clause.map do |v|
+        {
+          column: v["SORTBY"]["node"]["COLUMNREF"]["fields"].last,
+          direction: v["SORTBY"]["sortby_dir"] == 2 ? "desc" : "asc"
+        }
+      end
+    end
+
     def table_grant_commands(privilege, tables, user)
       tables.map do |table|
         "GRANT #{privilege} ON TABLE #{table} TO #{user}"