pghero_fork 2.7.3

data/lib/pghero/methods/replication.rb

@@ -0,0 +1,74 @@
+module PgHero
+  module Methods
+    module Replication
+      def replica?
+        unless defined?(@replica)
+          @replica = select_one("SELECT pg_is_in_recovery()")
+        end
+        @replica
+      end
+
+      # https://www.postgresql.org/message-id/CADKbJJWz9M0swPT3oqe8f9+tfD4-F54uE6Xtkh4nERpVsQnjnw@mail.gmail.com
+      def replication_lag
+        with_feature_support(:replication_lag) do
+          lag_condition =
+            if server_version_num >= 100000
+              "pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()"
+            else
+              "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
+            end
+
+          select_one <<-SQL
+            SELECT
+              CASE
+                WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
+                ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
+              END
+            AS replication_lag
+          SQL
+        end
+      end
+
+      def replication_slots
+        if server_version_num >= 90400
+          with_feature_support(:replication_slots, []) do
+            select_all <<-SQL
+              SELECT
+                slot_name,
+                database,
+                active
+              FROM pg_replication_slots
+            SQL
+          end
+        else
+          []
+        end
+      end
+
+      def replicating?
+        with_feature_support(:replicating?, false) do
+          select_all("SELECT state FROM pg_stat_replication").any?
+        end
+      end
+
+      private
+
+      def feature_support
+        @feature_support ||= {}
+      end
+
+      def with_feature_support(cache_key, default = nil)
+        # cache feature support to minimize errors in logs
+        return default if feature_support[cache_key] == false
+
+        begin
+          yield
+        rescue ActiveRecord::StatementInvalid => e
+          raise unless e.message.start_with?("PG::FeatureNotSupported:")
+          feature_support[cache_key] = false
+          default
+        end
+      end
+    end
+  end
+end

data/lib/pghero/methods/sequences.rb

@@ -0,0 +1,124 @@
+module PgHero
+  module Methods
+    module Sequences
+      def sequences
+        # get columns with default values
+        # use pg_get_expr to get correct default value
+        # it's what information_schema.columns uses
+        # also, exclude temporary tables to prevent error
+        # when accessing across sessions
+        sequences = select_all <<-SQL
+          SELECT
+            n.nspname AS table_schema,
+            c.relname AS table,
+            attname AS column,
+            format_type(a.atttypid, a.atttypmod) AS column_type,
+            pg_get_expr(d.adbin, d.adrelid) AS default_value
+          FROM
+            pg_catalog.pg_attribute a
+          INNER JOIN
+            pg_catalog.pg_class c ON c.oid = a.attrelid
+          INNER JOIN
+            pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+          INNER JOIN
+            pg_catalog.pg_attrdef d ON (a.attrelid, a.attnum) = (d.adrelid,  d.adnum)
+          WHERE
+            NOT a.attisdropped
+            AND a.attnum > 0
+            AND pg_get_expr(d.adbin, d.adrelid) LIKE 'nextval%'
+            AND n.nspname NOT LIKE 'pg\\_temp\\_%'
+        SQL
+
+        # parse out sequence
+        sequences.each do |column|
+          column[:max_value] = column[:column_type] == 'integer' ? 2147483647 : 9223372036854775807
+
+          column[:schema], column[:sequence] = parse_default_value(column[:default_value])
+          column.delete(:default_value) if column[:sequence]
+        end
+
+        add_sequence_attributes(sequences)
+
+        sequences.select { |s| s[:readable] }.each_slice(1024) do |slice|
+          sql = slice.map { |s| "SELECT last_value FROM #{quote_ident(s[:schema])}.#{quote_ident(s[:sequence])}" }.join(" UNION ALL ")
+
+          select_all(sql).zip(slice) do |row, seq|
+            seq[:last_value] = row[:last_value]
+          end
+        end
+
+        sequences.sort_by { |s| s[:sequence] }
+      end
+
+      def sequence_danger(threshold: 0.9, sequences: nil)
+        sequences ||= self.sequences
+        sequences.select { |s| s[:last_value] && s[:last_value] / s[:max_value].to_f > threshold }.sort_by { |s| s[:max_value] - s[:last_value] }
+      end
+
+      private
+
+      # can parse
+      # nextval('id_seq'::regclass)
+      # nextval(('id_seq'::text)::regclass)
+      def parse_default_value(default_value)
+        m = /^nextval\('(.+)'\:\:regclass\)$/.match(default_value)
+        m = /^nextval\(\('(.+)'\:\:text\)\:\:regclass\)$/.match(default_value) unless m
+        if m
+          unquote_ident(m[1])
+        else
+          []
+        end
+      end
+
+      def unquote_ident(value)
+        schema, seq = value.split(".")
+        unless seq
+          seq = schema
+          schema = nil
+        end
+        [unquote(schema), unquote(seq)]
+      end
+
+      # adds readable attribute to all sequences
+      # also adds schema if missing
+      def add_sequence_attributes(sequences)
+        # fetch data
+        sequence_attributes = select_all <<-SQL
+          SELECT
+            n.nspname AS schema,
+            c.relname AS sequence,
+            has_sequence_privilege(c.oid, 'SELECT') AS readable
+          FROM
+            pg_class c
+          INNER JOIN
+            pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+          WHERE
+            c.relkind = 'S'
+            AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+        SQL
+
+        # first populate missing schemas
+        missing_schema = sequences.select { |s| s[:schema].nil? && s[:sequence] }
+        if missing_schema.any?
+          sequence_schemas = sequence_attributes.group_by { |s| s[:sequence] }
+
+          missing_schema.each do |sequence|
+            schemas = sequence_schemas[sequence[:sequence]] || []
+
+            if schemas.size == 1
+              sequence[:schema] = schemas[0][:schema]
+            end
+            # otherwise, do nothing, will be marked as unreadable
+            # TODO better message for multiple schemas
+          end
+        end
+
+        # then populate attributes
+        readable = Hash[sequence_attributes.map { |s| [[s[:schema], s[:sequence]], s[:readable]] }]
+        sequences.each do |sequence|
+          sequence[:readable] = readable[[sequence[:schema], sequence[:sequence]]] || false
+        end
+      end
+    end
+  end
+end

data/lib/pghero/methods/settings.rb

@@ -0,0 +1,37 @@
+module PgHero
+  module Methods
+    module Settings
+      def settings
+        names =
+          if server_version_num >= 90500
+            %i(
+              max_connections shared_buffers effective_cache_size work_mem
+              maintenance_work_mem min_wal_size max_wal_size checkpoint_completion_target
+              wal_buffers default_statistics_target
+            )
+          else
+            %i(
+              max_connections shared_buffers effective_cache_size work_mem
+              maintenance_work_mem checkpoint_segments checkpoint_completion_target
+              wal_buffers default_statistics_target
+            )
+          end
+        fetch_settings(names)
+      end
+
+      def autovacuum_settings
+        fetch_settings %i(autovacuum autovacuum_max_workers autovacuum_vacuum_cost_limit autovacuum_vacuum_scale_factor autovacuum_analyze_scale_factor)
+      end
+
+      def vacuum_settings
+        fetch_settings %i(vacuum_cost_limit)
+      end
+
+      private
+
+      def fetch_settings(names)
+        Hash[names.map { |name| [name, select_one("SHOW #{name}")] }]
+      end
+    end
+  end
+end

data/lib/pghero/methods/space.rb

@@ -0,0 +1,141 @@
+module PgHero
+  module Methods
+    module Space
+      def database_size
+        PgHero.pretty_size select_one("SELECT pg_database_size(current_database())")
+      end
+
+      def relation_sizes
+        select_all_size <<-SQL
+          SELECT
+            n.nspname AS schema,
+            c.relname AS relation,
+            CASE WHEN c.relkind = 'r' THEN 'table' ELSE 'index' END AS type,
+            pg_table_size(c.oid) AS size_bytes
+          FROM
+            pg_class c
+          LEFT JOIN
+            pg_namespace n ON n.oid = c.relnamespace
+          WHERE
+            n.nspname NOT IN ('pg_catalog', 'information_schema')
+            AND n.nspname !~ '^pg_toast'
+            AND c.relkind IN ('r', 'i')
+          ORDER BY
+            pg_table_size(c.oid) DESC,
+            2 ASC
+        SQL
+      end
+
+      def table_sizes
+        select_all_size <<-SQL
+          SELECT
+            n.nspname AS schema,
+            c.relname AS table,
+            pg_total_relation_size(c.oid) AS size_bytes
+          FROM
+            pg_class c
+          LEFT JOIN
+            pg_namespace n ON n.oid = c.relnamespace
+          WHERE
+            n.nspname NOT IN ('pg_catalog', 'information_schema')
+            AND n.nspname !~ '^pg_toast'
+            AND c.relkind = 'r'
+          ORDER BY
+            pg_total_relation_size(c.oid) DESC,
+            2 ASC
+        SQL
+      end
+
+      def space_growth(days: 7, relation_sizes: nil)
+        if space_stats_enabled?
+          relation_sizes ||= self.relation_sizes
+          sizes = Hash[ relation_sizes.map { |r| [[r[:schema], r[:relation]], r[:size_bytes]] } ]
+          start_at = days.days.ago
+
+          stats = select_all_stats <<-SQL
+            WITH t AS (
+              SELECT
+                schema,
+                relation,
+                array_agg(size ORDER BY captured_at) AS sizes
+              FROM
+                pghero_space_stats
+              WHERE
+                database = #{quote(id)}
+                AND captured_at >= #{quote(start_at)}
+              GROUP BY
+                1, 2
+            )
+            SELECT
+              schema,
+              relation,
+              sizes[1] AS size_bytes
+            FROM
+              t
+            ORDER BY
+              1, 2
+          SQL
+
+          stats.each do |r|
+            relation = [r[:schema], r[:relation]]
+            if sizes[relation]
+              r[:growth_bytes] = sizes[relation] - r[:size_bytes]
+            end
+            r.delete(:size_bytes)
+          end
+          stats
+        else
+          raise NotEnabled, "Space stats not enabled"
+        end
+      end
+
+      def relation_space_stats(relation, schema: "public")
+        if space_stats_enabled?
+          relation_sizes ||= self.relation_sizes
+          sizes = Hash[ relation_sizes.map { |r| [[r[:schema], r[:relation]], r[:size_bytes]] } ]
+          start_at = 30.days.ago
+
+          stats = select_all_stats <<-SQL
+            SELECT
+              captured_at,
+              size AS size_bytes
+            FROM
+              pghero_space_stats
+            WHERE
+              database = #{quote(id)}
+              AND captured_at >= #{quote(start_at)}
+              AND schema = #{quote(schema)}
+              AND relation = #{quote(relation)}
+            ORDER BY
+              1 ASC
+          SQL
+
+          stats << {
+            captured_at: Time.now,
+            size_bytes: sizes[[schema, relation]].to_i
+          }
+        else
+          raise NotEnabled, "Space stats not enabled"
+        end
+      end
+
+      def capture_space_stats
+        now = Time.now
+        columns = %w(database schema relation size captured_at)
+        values = []
+        relation_sizes.each do |rs|
+          values << [id, rs[:schema], rs[:relation], rs[:size_bytes].to_i, now]
+        end
+        insert_stats("pghero_space_stats", columns, values) if values.any?
+      end
+
+      def clean_space_stats
+        PgHero::SpaceStats.where(database: id).where("captured_at < ?", 90.days.ago).delete_all
+      end
+
+      def space_stats_enabled?
+        table_exists?("pghero_space_stats")
+      end
+    end
+  end
+end

data/lib/pghero/methods/suggested_indexes.rb

@@ -0,0 +1,329 @@
+module PgHero
+  module Methods
+    module SuggestedIndexes
+      def suggested_indexes_enabled?
+        defined?(PgQuery) && Gem::Version.new(PgQuery::VERSION) >= Gem::Version.new("0.9.0") && query_stats_enabled?
+      end
+
+      # TODO clean this mess
+      def suggested_indexes_by_query(queries: nil, query_stats: nil, indexes: nil)
+        best_indexes = {}
+
+        if suggested_indexes_enabled?
+          # get most time-consuming queries
+          queries ||= (query_stats || self.query_stats(historical: true, start_at: 24.hours.ago)).map { |qs| qs[:query] }
+
+          # get best indexes for queries
+          best_indexes = best_index_helper(queries)
+
+          if best_indexes.any?
+            existing_columns = Hash.new { |hash, key| hash[key] = Hash.new { |hash2, key2| hash2[key2] = [] } }
+            indexes ||= self.indexes
+            indexes.group_by { |g| g[:using] }.each do |group, inds|
+              inds.each do |i|
+                existing_columns[group][i[:table]] << i[:columns]
+              end
+            end
+            indexes_by_table = indexes.group_by { |i| i[:table] }
+
+            best_indexes.each do |_query, best_index|
+              if best_index[:found]
+                index = best_index[:index]
+                best_index[:table_indexes] = indexes_by_table[index[:table]].to_a
+
+                # indexes of same type
+                indexes = existing_columns[index[:using] || "btree"][index[:table]]
+
+                if best_index[:structure][:sort].empty?
+                  # gist indexes without an opclass
+                  # (opclass is part of column name, so columns won't match if opclass present)
+                  indexes += existing_columns["gist"][index[:table]]
+
+                  # hash indexes work for equality
+                  indexes += existing_columns["hash"][index[:table]] if best_index[:structure][:where].all? { |v| v[:op] == "=" }
+
+                  # brin indexes work for all
+                  indexes += existing_columns["brin"][index[:table]]
+                end
+
+                covering_index = indexes.find { |e| index_covers?(e.map { |v| v.sub(/ inet_ops\z/, "") }, index[:columns]) }
+                if covering_index
+                  best_index[:covering_index] = covering_index
+                  best_index[:explanation] = "Covered by index on (#{covering_index.join(", ")})"
+                end
+              end
+            end
+          end
+        else
+          raise NotEnabled, "Suggested indexes not enabled"
+        end
+
+        best_indexes
+      end
+
+      def suggested_indexes(suggested_indexes_by_query: nil, **options)
+        indexes = []
+
+        (suggested_indexes_by_query || self.suggested_indexes_by_query(**options)).select { |_s, i| i[:found] && !i[:covering_index] }.group_by { |_s, i| i[:index] }.each do |index, group|
+          details = {}
+          group.map(&:second).each do |g|
+            details = details.except(:index).deep_merge(g)
+          end
+          indexes << index.merge(queries: group.map(&:first), details: details)
+        end
+
+        indexes.sort_by { |i| [i[:table], i[:columns]] }
+      end
+
+      def autoindex(create: false)
+        suggested_indexes.each do |index|
+          p index
+          if create
+            connection.execute("CREATE INDEX CONCURRENTLY ON #{quote_table_name(index[:table])} (#{index[:columns].map { |c| quote_table_name(c) }.join(",")})")
+          end
+        end
+      end
+
+      def best_index(statement)
+        best_index_helper([statement])[statement]
+      end
+
+      private
+
+      def best_index_helper(statements)
+        indexes = {}
+
+        # see if this is a query we understand and can use
+        parts = {}
+        statements.each do |statement|
+          parts[statement] = best_index_structure(statement)
+        end
+
+        # get stats about columns for relevant tables
+        tables = parts.values.map { |t| t[:table] }.uniq
+        # TODO get schema from query structure, then try search path
+        schema = PgHero.connection_config(connection_model)[:schema] || "public"
+        if tables.any?
+          row_stats = Hash[table_stats(table: tables, schema: schema).map { |i| [i[:table], i[:estimated_rows]] }]
+          col_stats = column_stats(table: tables, schema: schema).group_by { |i| i[:table] }
+        end
+
+        # find best index based on query structure and column stats
+        parts.each do |statement, structure|
+          index = {found: false}
+
+          if structure[:error]
+            index[:explanation] = structure[:error]
+          elsif structure[:table].start_with?("pg_")
+            index[:explanation] = "System table"
+          else
+            index[:structure] = structure
+
+            table = structure[:table]
+            where = structure[:where].uniq
+            sort = structure[:sort]
+
+            total_rows = row_stats[table].to_i
+            index[:rows] = total_rows
+
+            ranks = Hash[col_stats[table].to_a.map { |r| [r[:column], r] }]
+            columns = (where + sort).map { |c| c[:column] }.uniq
+
+            if columns.any?
+              if columns.all? { |c| ranks[c] }
+                first_desc = sort.index { |c| c[:direction] == "desc" }
+                sort = sort.first(first_desc + 1) if first_desc
+                where = where.sort_by { |c| [row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]), c[:column]] } + sort
+
+                index[:row_estimates] = Hash[where.map { |c| ["#{c[:column]} (#{c[:op] || "sort"})", row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]).round] }]
+
+                # no index needed if less than 500 rows
+                if total_rows >= 500
+
+                  if ["~~", "~~*"].include?(where.first[:op])
+                    index[:found] = true
+                    index[:row_progression] = [total_rows, index[:row_estimates].values.first]
+                    index[:index] = {table: table, columns: ["#{where.first[:column]} gist_trgm_ops"], using: "gist"}
+                  else
+                    # if most values are unique, no need to index others
+                    rows_left = total_rows
+                    final_where = []
+                    prev_rows_left = [rows_left]
+                    where.reject { |c| ["~~", "~~*"].include?(c[:op]) }.each do |c|
+                      next if final_where.include?(c[:column])
+                      final_where << c[:column]
+                      rows_left = row_estimates(ranks[c[:column]], total_rows, rows_left, c[:op])
+                      prev_rows_left << rows_left
+                      if rows_left < 50 || final_where.size >= 2 || [">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(c[:op])
+                        break
+                      end
+                    end
+
+                    index[:row_progression] = prev_rows_left.map(&:round)
+
+                    # if the last indexes don't give us much, don't include
+                    prev_rows_left.reverse!
+                    (prev_rows_left.size - 1).times do |i|
+                      if prev_rows_left[i] > prev_rows_left[i + 1] * 0.3
+                        final_where.pop
+                      else
+                        break
+                      end
+                    end
+
+                    if final_where.any?
+                      index[:found] = true
+                      index[:index] = {table: table, columns: final_where}
+                    end
+                  end
+                else
+                  index[:explanation] = "No index needed if less than 500 rows"
+                end
+              else
+                index[:explanation] = "Stats not found"
+              end
+            else
+              index[:explanation] = "No columns to index"
+            end
+          end
+
+          indexes[statement] = index
+        end
+
+        indexes
+      end
+
+      def best_index_structure(statement)
+        return {error: "Too large"} if statement.to_s.length > 10000
+
+        begin
+          tree = PgQuery.parse(statement).tree
+        rescue PgQuery::ParseError
+          return {error: "Parse error"}
+        end
+        return {error: "Unknown structure"} unless tree.size == 1
+
+        tree = tree.first
+
+        # pg_query 1.0.0
+        tree = tree["RawStmt"]["stmt"] if tree["RawStmt"]
+
+        table = parse_table(tree) rescue nil
+        unless table
+          error =
+            case tree.keys.first
+            when "InsertStmt"
+              "INSERT statement"
+            when "VariableSetStmt"
+              "SET statement"
+            when "SelectStmt"
+              if (tree["SelectStmt"]["fromClause"].first["JoinExpr"] rescue false)
+                "JOIN not supported yet"
+              end
+            end
+          return {error: error || "Unknown structure"}
+        end
+
+        select = tree.values.first
+        where = (select["whereClause"] ? parse_where(select["whereClause"]) : []) rescue nil
+        return {error: "Unknown structure"} unless where
+
+        sort = (select["sortClause"] ? parse_sort(select["sortClause"]) : []) rescue []
+
+        {table: table, where: where, sort: sort}
+      end
+
+      # TODO better row estimation
+      # https://www.postgresql.org/docs/current/static/row-estimation-examples.html
+      def row_estimates(stats, total_rows, rows_left, op)
+        case op
+        when "null"
+          rows_left * stats[:null_frac].to_f
+        when "not_null"
+          rows_left * (1 - stats[:null_frac].to_f)
+        else
+          rows_left *= (1 - stats[:null_frac].to_f)
+          ret =
+            if stats[:n_distinct].to_f == 0
+              0
+            elsif stats[:n_distinct].to_f < 0
+              if total_rows > 0
+                (-1 / stats[:n_distinct].to_f) * (rows_left / total_rows.to_f)
+              else
+                0
+              end
+            else
+              rows_left / stats[:n_distinct].to_f
+            end
+
+          case op
+          when ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"
+            (rows_left + ret) / 10.0 # TODO better approximation
+          when "<>"
+            rows_left - ret
+          else
+            ret
+          end
+        end
+      end
+
+      def parse_table(tree)
+        case tree.keys.first
+        when "SelectStmt"
+          tree["SelectStmt"]["fromClause"].first["RangeVar"]["relname"]
+        when "DeleteStmt"
+          tree["DeleteStmt"]["relation"]["RangeVar"]["relname"]
+        when "UpdateStmt"
+          tree["UpdateStmt"]["relation"]["RangeVar"]["relname"]
+        end
+      end
+
+      # TODO capture values
+      def parse_where(tree)
+        aexpr = tree["A_Expr"]
+
+        if tree["BoolExpr"]
+          if tree["BoolExpr"]["boolop"] == 0
+            tree["BoolExpr"]["args"].flat_map { |v| parse_where(v) }
+          else
+            raise "Not Implemented"
+          end
+        elsif aexpr && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(aexpr["name"].first["String"]["str"])
+          [{column: aexpr["lexpr"]["ColumnRef"]["fields"].last["String"]["str"], op: aexpr["name"].first["String"]["str"]}]
+        elsif tree["NullTest"]
+          op = tree["NullTest"]["nulltesttype"] == 1 ? "not_null" : "null"
+          [{column: tree["NullTest"]["arg"]["ColumnRef"]["fields"].last["String"]["str"], op: op}]
+        else
+          raise "Not Implemented"
+        end
+      end
+
+      def parse_sort(sort_clause)
+        sort_clause.map do |v|
+          {
+            column: v["SortBy"]["node"]["ColumnRef"]["fields"].last["String"]["str"],
+            direction: v["SortBy"]["sortby_dir"] == 2 ? "desc" : "asc"
+          }
+        end
+      end
+
+      def column_stats(schema: nil, table: nil)
+        select_all <<-SQL
+          SELECT
+            schemaname AS schema,
+            tablename AS table,
+            attname AS column,
+            null_frac,
+            n_distinct
+          FROM
+            pg_stats
+          WHERE
+            schemaname = #{quote(schema)}
+            #{table ? "AND tablename IN (#{Array(table).map { |t| quote(t) }.join(", ")})" : ""}
+          ORDER BY
+            1, 2, 3
+        SQL
+      end
+    end
+  end
+end