pghero 1.2.2 → 1.2.3

data/lib/pghero/methods/maintenance.rb

@@ -0,0 +1,61 @@
+module PgHero
+  module Methods
+    module Maintenance
+      # http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND
+      # "the system will shut down and refuse to start any new transactions
+      # once there are fewer than 1 million transactions left until wraparound"
+      # warn when 10,000,000 transactions left
+      def transaction_id_danger(options = {})
+        threshold = options[:threshold] || 10000000
+        select_all <<-SQL
+          SELECT
+            c.oid::regclass::text AS table,
+            2146483648 - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) AS transactions_before_shutdown
+          FROM
+            pg_class c
+          LEFT JOIN
+            pg_class t ON c.reltoastrelid = t.oid
+          WHERE
+            c.relkind = 'r'
+            AND (2146483648 - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid))) < #{threshold}
+          ORDER BY
+           2, 1
+        SQL
+      end
+
+      def autovacuum_danger
+        select_all <<-SQL
+          SELECT
+            c.oid::regclass::text as table,
+            (SELECT setting FROM pg_settings WHERE name = 'autovacuum_freeze_max_age')::int -
+            GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) AS transactions_before_autovacuum
+          FROM
+            pg_class c
+          LEFT JOIN
+            pg_class t ON c.reltoastrelid = t.oid
+          WHERE
+            c.relkind = 'r'
+            AND (SELECT setting FROM pg_settings WHERE name = 'autovacuum_freeze_max_age')::int - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) < 2000000
+          ORDER BY
+            transactions_before_autovacuum
+        SQL
+      end
+
+      def maintenance_info
+        select_all <<-SQL
+          SELECT
+            schemaname AS schema,
+            relname AS table,
+            last_vacuum,
+            last_autovacuum,
+            last_analyze,
+            last_autoanalyze
+          FROM
+            pg_stat_user_tables
+          ORDER BY
+            1, 2
+        SQL
+      end
+    end
+  end
+end

data/lib/pghero/methods/queries.rb

@@ -0,0 +1,73 @@
+module PgHero
+  module Methods
+    module Queries
+      def running_queries
+        select_all <<-SQL
+          SELECT
+            pid,
+            state,
+            application_name AS source,
+            age(now(), xact_start) AS duration,
+            waiting,
+            query,
+            xact_start AS started_at
+          FROM
+            pg_stat_activity
+          WHERE
+            query <> '<insufficient privilege>'
+            AND state <> 'idle'
+            AND pid <> pg_backend_pid()
+          ORDER BY
+            query_start DESC
+        SQL
+      end
+
+      def long_running_queries
+        select_all <<-SQL
+          SELECT
+            pid,
+            state,
+            application_name AS source,
+            age(now(), xact_start) AS duration,
+            waiting,
+            query,
+            xact_start AS started_at
+          FROM
+            pg_stat_activity
+          WHERE
+            query <> '<insufficient privilege>'
+            AND state <> 'idle'
+            AND pid <> pg_backend_pid()
+            AND now() - query_start > interval '#{long_running_query_sec.to_i} seconds'
+          ORDER BY
+            query_start DESC
+        SQL
+      end
+
+      def slow_queries(options = {})
+        query_stats = options[:query_stats] || self.query_stats(options.except(:query_stats))
+        query_stats.select { |q| q["calls"].to_i >= slow_query_calls.to_i && q["average_time"].to_i >= slow_query_ms.to_i }
+      end
+
+      def locks
+        select_all <<-SQL
+          SELECT DISTINCT ON (pid)
+            pg_stat_activity.pid,
+            pg_stat_activity.query,
+            age(now(), pg_stat_activity.query_start) AS age
+          FROM
+            pg_stat_activity
+          INNER JOIN
+            pg_locks ON pg_locks.pid = pg_stat_activity.pid
+          WHERE
+            pg_stat_activity.query <> '<insufficient privilege>'
+            AND pg_locks.mode = 'ExclusiveLock'
+            AND pg_stat_activity.pid <> pg_backend_pid()
+          ORDER BY
+            pid,
+            query_start
+        SQL
+      end
+    end
+  end
+end

data/lib/pghero/methods/query_stats.rb

@@ -0,0 +1,188 @@
+module PgHero
+  module Methods
+    module QueryStats
+      def query_stats(options = {})
+        current_query_stats = (options[:historical] && options[:end_at] && options[:end_at] < Time.now ? [] : current_query_stats(options)).index_by { |q| q["query"] }
+        historical_query_stats = (options[:historical] ? historical_query_stats(options) : []).index_by { |q| q["query"] }
+        current_query_stats.default = {}
+        historical_query_stats.default = {}
+
+        query_stats = []
+        (current_query_stats.keys + historical_query_stats.keys).uniq.each do |query|
+          value = {
+            "query" => query,
+            "total_minutes" => current_query_stats[query]["total_minutes"].to_f + historical_query_stats[query]["total_minutes"].to_f,
+            "calls" => current_query_stats[query]["calls"].to_i + historical_query_stats[query]["calls"].to_i
+          }
+          value["average_time"] = value["total_minutes"] * 1000 * 60 / value["calls"]
+          value["total_percent"] = value["total_minutes"] * 100.0 / (current_query_stats[query]["all_queries_total_minutes"].to_f + historical_query_stats[query]["all_queries_total_minutes"].to_f)
+          query_stats << value
+        end
+        sort = options[:sort] || "total_minutes"
+        query_stats = query_stats.sort_by { |q| -q[sort] }.first(100)
+        if options[:min_average_time]
+          query_stats.reject! { |q| q["average_time"].to_f < options[:min_average_time] }
+        end
+        if options[:min_calls]
+          query_stats.reject! { |q| q["calls"].to_i < options[:min_calls] }
+        end
+        query_stats
+      end
+
+      def query_stats_available?
+        select_all("SELECT COUNT(*) AS count FROM pg_available_extensions WHERE name = 'pg_stat_statements'").first["count"].to_i > 0
+      end
+
+      def query_stats_enabled?
+        select_all("SELECT COUNT(*) AS count FROM pg_extension WHERE extname = 'pg_stat_statements'").first["count"].to_i > 0 && query_stats_readable?
+      end
+
+      def query_stats_readable?
+        select_all("SELECT has_table_privilege(current_user, 'pg_stat_statements', 'SELECT')").first["has_table_privilege"] == "t"
+      rescue ActiveRecord::StatementInvalid
+        false
+      end
+
+      def enable_query_stats
+        execute("CREATE EXTENSION pg_stat_statements")
+      end
+
+      def disable_query_stats
+        execute("DROP EXTENSION IF EXISTS pg_stat_statements")
+        true
+      end
+
+      def reset_query_stats
+        if query_stats_enabled?
+          execute("SELECT pg_stat_statements_reset()")
+          true
+        else
+          false
+        end
+      end
+
+      def capture_query_stats
+        config["databases"].keys.each do |database|
+          with(database) do
+            now = Time.now
+            query_stats = self.query_stats(limit: 1000000)
+            if query_stats.any? && reset_query_stats
+              values =
+                query_stats.map do |qs|
+                  [
+                    database,
+                    qs["query"],
+                    qs["total_minutes"].to_f * 60 * 1000,
+                    qs["calls"],
+                    now
+                  ].map { |v| quote(v) }.join(",")
+                end.map { |v| "(#{v})" }.join(",")
+
+              stats_connection.execute("INSERT INTO pghero_query_stats (database, query, total_time, calls, captured_at) VALUES #{values}")
+            end
+          end
+        end
+      end
+
+      # http://stackoverflow.com/questions/20582500/how-to-check-if-a-table-exists-in-a-given-schema
+      def historical_query_stats_enabled?
+        # TODO use schema from config
+        stats_connection.select_all(squish <<-SQL
+          SELECT EXISTS (
+            SELECT
+              1
+            FROM
+              pg_catalog.pg_class c
+            INNER JOIN
+              pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+            WHERE
+              n.nspname = 'public'
+              AND c.relname = 'pghero_query_stats'
+              AND c.relkind = 'r'
+          )
+        SQL
+        ).to_a.first["exists"] == "t"
+      end
+
+      def stats_connection
+        ::PgHero::QueryStats.connection
+      end
+
+      private
+
+      # http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
+      def current_query_stats(options = {})
+        if query_stats_enabled?
+          limit = options[:limit] || 100
+          sort = options[:sort] || "total_minutes"
+          select_all <<-SQL
+            WITH query_stats AS (
+              SELECT
+                query,
+                (total_time / 1000 / 60) as total_minutes,
+                (total_time / calls) as average_time,
+                calls
+              FROM
+                pg_stat_statements
+              INNER JOIN
+                pg_database ON pg_database.oid = pg_stat_statements.dbid
+              WHERE
+                pg_database.datname = current_database()
+            )
+            SELECT
+              query,
+              total_minutes,
+              average_time,
+              calls,
+              total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent,
+              (SELECT SUM(total_minutes) FROM query_stats) AS all_queries_total_minutes
+            FROM
+              query_stats
+            ORDER BY
+              #{quote_table_name(sort)} DESC
+            LIMIT #{limit.to_i}
+          SQL
+        else
+          []
+        end
+      end
+
+      def historical_query_stats(options = {})
+        if historical_query_stats_enabled?
+          sort = options[:sort] || "total_minutes"
+          stats_connection.select_all squish <<-SQL
+            WITH query_stats AS (
+              SELECT
+                query,
+                (SUM(total_time) / 1000 / 60) as total_minutes,
+                (SUM(total_time) / SUM(calls)) as average_time,
+                SUM(calls) as calls
+              FROM
+                pghero_query_stats
+              WHERE
+                database = #{quote(current_database)}
+                #{options[:start_at] ? "AND captured_at >= #{quote(options[:start_at])}" : ""}
+                #{options[:end_at] ? "AND captured_at <= #{quote(options[:end_at])}" : ""}
+              GROUP BY
+                query
+            )
+            SELECT
+              query,
+              total_minutes,
+              average_time,
+              calls,
+              total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent,
+              (SELECT SUM(total_minutes) FROM query_stats) AS all_queries_total_minutes
+            FROM
+              query_stats
+            ORDER BY
+              #{quote_table_name(sort)} DESC
+            LIMIT 100
+          SQL
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/pghero/methods/replica.rb

@@ -0,0 +1,22 @@
+module PgHero
+  module Methods
+    module Replica
+      def replica?
+        select_all("SELECT setting FROM pg_settings WHERE name = 'hot_standby'").first["setting"] == "on"
+      end
+
+      # http://www.postgresql.org/message-id/CADKbJJWz9M0swPT3oqe8f9+tfD4-F54uE6Xtkh4nERpVsQnjnw@mail.gmail.com
+      def replication_lag
+        select_all(<<-SQL
+          SELECT
+            CASE
+              WHEN pg_last_xlog_receive_location() = pg_last_xlog_replay_location() THEN 0
+              ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
+            END
+          AS replication_lag
+        SQL
+        ).first["replication_lag"].to_f
+      end
+    end
+  end
+end

data/lib/pghero/methods/space.rb

@@ -0,0 +1,30 @@
+module PgHero
+  module Methods
+    module Space
+      def database_size
+        select_all("SELECT pg_size_pretty(pg_database_size(current_database()))").first["pg_size_pretty"]
+      end
+
+      def relation_sizes
+        select_all <<-SQL
+          SELECT
+            n.nspname AS schema,
+            c.relname AS name,
+            CASE WHEN c.relkind = 'r' THEN 'table' ELSE 'index' END AS type,
+            pg_size_pretty(pg_table_size(c.oid)) AS size
+          FROM
+            pg_class c
+          LEFT JOIN
+            pg_namespace n ON (n.oid = c.relnamespace)
+          WHERE
+            n.nspname NOT IN ('pg_catalog', 'information_schema')
+            AND n.nspname !~ '^pg_toast'
+            AND c.relkind IN ('r', 'i')
+          ORDER BY
+            pg_table_size(c.oid) DESC,
+            name ASC
+        SQL
+      end
+    end
+  end
+end

data/lib/pghero/methods/suggested_indexes.rb

@@ -0,0 +1,322 @@
+module PgHero
+  module Methods
+    module SuggestedIndexes
+     def suggested_indexes_enabled?
+        defined?(PgQuery) && query_stats_enabled?
+      end
+
+      # TODO clean this mess
+      def suggested_indexes_by_query(options = {})
+        best_indexes = {}
+
+        if suggested_indexes_enabled?
+          # get most time-consuming queries
+          queries = options[:queries] || (options[:query_stats] || query_stats(historical: true, start_at: 24.hours.ago)).map { |qs| qs["query"] }
+
+          # get best indexes for queries
+          best_indexes = best_index_helper(queries)
+
+          if best_indexes.any?
+            existing_columns = Hash.new { |hash, key| hash[key] = Hash.new { |hash2, key2| hash2[key2] = [] } }
+            indexes = self.indexes
+            indexes.group_by { |g| g["using"] }.each do |group, inds|
+              inds.each do |i|
+                existing_columns[group][i["table"]] << i["columns"]
+              end
+            end
+            indexes_by_table = indexes.group_by { |i| i["table"] }
+
+            best_indexes.each do |_query, best_index|
+              if best_index[:found]
+                index = best_index[:index]
+                best_index[:table_indexes] = indexes_by_table[index[:table]].to_a
+                covering_index = existing_columns[index[:using] || "btree"][index[:table]].find { |e| index_covers?(e, index[:columns]) }
+                if covering_index
+                  best_index[:covering_index] = covering_index
+                  best_index[:explanation] = "Covered by index on (#{covering_index.join(", ")})"
+                end
+              end
+            end
+          end
+        end
+
+        best_indexes
+      end
+
+      def suggested_indexes(options = {})
+        indexes = []
+
+        (options[:suggested_indexes_by_query] || suggested_indexes_by_query(options)).select { |_s, i| i[:found] && !i[:covering_index] }.group_by { |_s, i| i[:index] }.each do |index, group|
+          details = {}
+          group.map(&:second).each do |g|
+            details = details.except(:index).deep_merge(g)
+          end
+          indexes << index.merge(queries: group.map(&:first), details: details)
+        end
+
+        indexes.sort_by { |i| [i[:table], i[:columns]] }
+      end
+
+      def autoindex(options = {})
+        suggested_indexes.each do |index|
+          p index
+          if options[:create]
+            connection.execute("CREATE INDEX CONCURRENTLY ON #{quote_table_name(index[:table])} (#{index[:columns].map { |c| quote_table_name(c) }.join(",")})")
+          end
+        end
+      end
+
+      def autoindex_all(options = {})
+        config["databases"].keys.each do |database|
+          with(database) do
+            puts "Autoindexing #{database}..."
+            autoindex(options)
+          end
+        end
+      end
+
+      def best_index(statement, _options = {})
+        best_index_helper([statement])[statement]
+      end
+
+      private
+
+      def best_index_helper(statements)
+        indexes = {}
+
+        # see if this is a query we understand and can use
+        parts = {}
+        statements.each do |statement|
+          parts[statement] = best_index_structure(statement)
+        end
+
+        # get stats about columns for relevant tables
+        tables = parts.values.map { |t| t[:table] }.uniq
+        # TODO get schema from query structure, then try search path
+        schema = connection_model.connection_config[:schema] || "public"
+        if tables.any?
+          row_stats = Hash[table_stats(table: tables, schema: schema).map { |i| [i["table"], i["reltuples"]] }]
+          col_stats = column_stats(table: tables, schema: schema).group_by { |i| i["table"] }
+        end
+
+        # find best index based on query structure and column stats
+        parts.each do |statement, structure|
+          index = {found: false}
+
+          if structure[:error]
+            index[:explanation] = structure[:error]
+          elsif structure[:table].start_with?("pg_")
+            index[:explanation] = "System table"
+          else
+            index[:structure] = structure
+
+            table = structure[:table]
+            where = structure[:where].uniq
+            sort = structure[:sort]
+
+            total_rows = row_stats[table].to_i
+            index[:rows] = total_rows
+
+            ranks = Hash[col_stats[table].to_a.map { |r| [r["column"], r] }]
+            columns = (where + sort).map { |c| c[:column] }.uniq
+
+            if columns.any?
+              if columns.all? { |c| ranks[c] }
+                first_desc = sort.index { |c| c[:direction] == "desc" }
+                sort = sort.first(first_desc + 1) if first_desc
+                where = where.sort_by { |c| [row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]), c[:column]] } + sort
+
+                index[:row_estimates] = Hash[where.map { |c| ["#{c[:column]} (#{c[:op] || "sort"})", row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]).round] }]
+
+                # no index needed if less than 500 rows
+                if total_rows >= 500
+
+                  if ["~~", "~~*"].include?(where.first[:op])
+                    index[:found] = true
+                    index[:row_progression] = [total_rows, index[:row_estimates].values.first]
+                    index[:index] = {table: table, columns: ["#{where.first[:column]} gist_trgm_ops"], using: "gist"}
+                  else
+                    # if most values are unique, no need to index others
+                    rows_left = total_rows
+                    final_where = []
+                    prev_rows_left = [rows_left]
+                    where.reject { |c| ["~~", "~~*"].include?(c[:op]) }.each do |c|
+                      next if final_where.include?(c[:column])
+                      final_where << c[:column]
+                      rows_left = row_estimates(ranks[c[:column]], total_rows, rows_left, c[:op])
+                      prev_rows_left << rows_left
+                      if rows_left < 50 || final_where.size >= 2 || [">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(c[:op])
+                        break
+                      end
+                    end
+
+                    index[:row_progression] = prev_rows_left.map(&:round)
+
+                    # if the last indexes don't give us much, don't include
+                    prev_rows_left.reverse!
+                    (prev_rows_left.size - 1).times do |i|
+                      if prev_rows_left[i] > prev_rows_left[i + 1] * 0.3
+                        final_where.pop
+                      else
+                        break
+                      end
+                    end
+
+                    if final_where.any?
+                      index[:found] = true
+                      index[:index] = {table: table, columns: final_where}
+                    end
+                  end
+                else
+                  index[:explanation] = "No index needed if less than 500 rows"
+                end
+              else
+                index[:explanation] = "Stats not found"
+              end
+            else
+              index[:explanation] = "No columns to index"
+            end
+          end
+
+          indexes[statement] = index
+        end
+
+        indexes
+      end
+
+      def best_index_structure(statement)
+        begin
+          tree = PgQuery.parse(statement).parsetree
+        rescue PgQuery::ParseError
+          return {error: "Parse error"}
+        end
+        return {error: "Unknown structure"} unless tree.size == 1
+
+        tree = tree.first
+        table = parse_table(tree) rescue nil
+        unless table
+          error =
+            case tree.keys.first
+            when "INSERT INTO"
+              "INSERT statement"
+            when "SET"
+              "SET statement"
+            when "SELECT"
+              if (tree["SELECT"]["fromClause"].first["JOINEXPR"] rescue false)
+                "JOIN not supported yet"
+              end
+            end
+          return {error: error || "Unknown structure"}
+        end
+
+        select = tree["SELECT"] || tree["DELETE FROM"] || tree["UPDATE"]
+        where = (select["whereClause"] ? parse_where(select["whereClause"]) : []) rescue nil
+        return {error: "Unknown structure"} unless where
+
+        sort = (select["sortClause"] ? parse_sort(select["sortClause"]) : []) rescue []
+
+        {table: table, where: where, sort: sort}
+       end
+
+      def index_covers?(indexed_columns, columns)
+        indexed_columns.first(columns.size) == columns
+      end
+
+      # TODO better row estimation
+      # http://www.postgresql.org/docs/current/static/row-estimation-examples.html
+      def row_estimates(stats, total_rows, rows_left, op)
+        case op
+        when "null"
+          rows_left * stats["null_frac"].to_f
+        when "not_null"
+          rows_left * (1 - stats["null_frac"].to_f)
+        else
+          rows_left *= (1 - stats["null_frac"].to_f)
+          ret =
+            if stats["n_distinct"].to_f == 0
+              0
+            elsif stats["n_distinct"].to_f < 0
+              if total_rows > 0
+                (-1 / stats["n_distinct"].to_f) * (rows_left / total_rows.to_f)
+              else
+                0
+              end
+            else
+              rows_left / stats["n_distinct"].to_f
+            end
+
+          case op
+          when ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"
+            (rows_left + ret) / 10.0 # TODO better approximation
+          when "<>"
+            rows_left - ret
+          else
+            ret
+          end
+        end
+      end
+
+      def parse_table(tree)
+        case tree.keys.first
+        when "SELECT"
+          tree["SELECT"]["fromClause"].first["RANGEVAR"]["relname"]
+        when "DELETE FROM"
+          tree["DELETE FROM"]["relation"]["RANGEVAR"]["relname"]
+        when "UPDATE"
+          tree["UPDATE"]["relation"]["RANGEVAR"]["relname"]
+        end
+      end
+
+      # TODO capture values
+      def parse_where(tree)
+        aexpr = tree["AEXPR"] || tree[nil]
+
+        if tree["BOOLEXPR"]
+          if tree["BOOLEXPR"]["boolop"] == 0
+            tree["BOOLEXPR"]["args"].flat_map { |v| parse_where(v) }
+          end
+        elsif tree["AEXPR AND"]
+          left = parse_where(tree["AEXPR AND"]["lexpr"])
+          right = parse_where(tree["AEXPR AND"]["rexpr"])
+          left + right if left && right
+        elsif aexpr && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(aexpr["name"].first)
+          [{column: aexpr["lexpr"]["COLUMNREF"]["fields"].last, op: aexpr["name"].first}]
+        elsif tree["AEXPR IN"] && ["=", "<>"].include?(tree["AEXPR IN"]["name"].first)
+          [{column: tree["AEXPR IN"]["lexpr"]["COLUMNREF"]["fields"].last, op: tree["AEXPR IN"]["name"].first}]
+        elsif tree["NULLTEST"]
+          op = tree["NULLTEST"]["nulltesttype"] == 1 ? "not_null" : "null"
+          [{column: tree["NULLTEST"]["arg"]["COLUMNREF"]["fields"].last, op: op}]
+        end
+      end
+
+      def parse_sort(sort_clause)
+        sort_clause.map do |v|
+          {
+            column: v["SORTBY"]["node"]["COLUMNREF"]["fields"].last,
+            direction: v["SORTBY"]["sortby_dir"] == 2 ? "desc" : "asc"
+          }
+        end
+      end
+
+      def column_stats(options = {})
+        schema = options[:schema]
+        tables = options[:table] ? Array(options[:table]) : nil
+        select_all <<-SQL
+          SELECT
+            schemaname AS schema,
+            tablename AS table,
+            attname AS column,
+            null_frac,
+            n_distinct
+          FROM
+            pg_stats
+          WHERE
+        #{tables ? "tablename IN (#{tables.map { |t| quote(t) }.join(", ")})" : "1 = 1"}
+            AND schemaname = #{quote(schema)}
+          ORDER BY
+            1, 2, 3
+        SQL
+      end
+    end
+  end
+end