pghero 3.1.0 → 3.7.0

data/lib/pghero/methods/basic.rb

@@ -31,21 +31,26 @@ module PgHero
       end
 
       def quote_ident(value)
-        connection.quote_column_name(value)
+        with_connection { |c| c.quote_column_name(value) }
       end
 
       private
 
-      def select_all(sql, conn: nil, query_columns: [])
-        conn ||= connection
+      def select_all(sql, stats: false, query_columns: [])
+        with_connection(stats: stats) do |conn|
+          select_all_leased(sql, conn: conn, query_columns: query_columns)
+        end
+      end
+
+      def select_all_leased(sql, conn:, query_columns:)
         # squish for logs
         retries = 0
         begin
           result = conn.select_all(add_source(squish(sql)))
-          if ActiveRecord::VERSION::STRING.to_f >= 6.1
-            result = result.map(&:symbolize_keys)
+          if ActiveRecord::VERSION::MAJOR >= 8
+            result = result.to_a.map(&:symbolize_keys)
           else
-            result = result.map { |row| row.to_h { |col, val| [col.to_sym, result.column_types[col].send(:cast_value, val)] } }
+            result = result.map(&:symbolize_keys)
           end
           if filter_data
             query_columns.each do |column|
@@ -81,7 +86,7 @@ module PgHero
       end
 
       def select_all_stats(sql, **options)
-        select_all(sql, **options, conn: stats_connection)
+        select_all(sql, **options, stats: true)
       end
 
       def select_all_size(sql)
@@ -92,35 +97,21 @@ module PgHero
         result
       end
 
-      def select_one(sql, conn: nil)
-        select_all(sql, conn: conn).first.values.first
-      end
-
-      def select_one_stats(sql)
-        select_one(sql, conn: stats_connection)
+      def select_one(sql)
+        select_all(sql).first.values.first
       end
 
       def execute(sql)
-        connection.execute(add_source(sql))
+        with_connection { |c| c.execute(add_source(sql)) }
       end
 
-      def connection
-        connection_model.connection
+      def with_connection(stats: false, &block)
+        model = stats ? ::PgHero::Stats : connection_model
+        model.connection_pool.with_connection(&block)
       end
 
-      def stats_connection
-        ::PgHero::Stats.connection
-      end
-
-      def insert_stats(table, columns, values)
-        values = values.map { |v| "(#{v.map { |v2| quote(v2) }.join(",")})" }.join(",")
-        columns = columns.map { |v| quote_table_name(v) }.join(",")
-        stats_connection.execute("INSERT INTO #{quote_table_name(table)} (#{columns}) VALUES #{values}")
-      end
-
-      # from ActiveSupport
       def squish(str)
-        str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
+        str.to_s.squish
       end
 
       def add_source(sql)
@@ -128,11 +119,15 @@ module PgHero
       end
 
       def quote(value)
-        connection.quote(value)
+        with_connection { |c| c.quote(value) }
       end
 
       def quote_table_name(value)
-        connection.quote_table_name(value)
+        with_connection { |c| c.quote_table_name(value) }
+      end
+
+      def quote_column_name(value)
+        with_connection { |c| c.quote_column_name(value) }
       end
 
       def unquote(part)
@@ -153,7 +148,7 @@ module PgHero
       end
 
       def table_exists?(table)
-        stats_connection.table_exists?(table)
+        with_connection(stats: true) { |c| c.table_exists?(table) }
       end
     end
   end

data/lib/pghero/methods/connections.rb

@@ -3,7 +3,7 @@ module PgHero
     module Connections
       def connections
         if server_version_num >= 90500
-          select_all <<-SQL
+          select_all <<~SQL
             SELECT
               pg_stat_activity.pid,
               datname AS database,
@@ -20,7 +20,7 @@ module PgHero
               pg_stat_activity.pid
           SQL
         else
-          select_all <<-SQL
+          select_all <<~SQL
             SELECT
               pid,
               datname AS database,
@@ -41,7 +41,7 @@ module PgHero
       end
 
       def connection_states
-        states = select_all <<-SQL
+        states = select_all <<~SQL
           SELECT
             state,
             COUNT(*) AS connections
@@ -57,7 +57,7 @@ module PgHero
       end
 
       def connection_sources
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             datname AS database,
             usename AS user,

data/lib/pghero/methods/constraints.rb

@@ -4,7 +4,7 @@ module PgHero
       # referenced fields can be nil
       # as not all constraints are foreign keys
       def invalid_constraints
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             nsp.nspname AS schema,
             rel.relname AS table,

data/lib/pghero/methods/explain.rb

@@ -9,10 +9,10 @@ module PgHero
 
         # use transaction for safety
         with_transaction(statement_timeout: (explain_timeout_sec * 1000).round, rollback: true) do
-          if (sql.sub(/;\z/, "").include?(";") || sql.upcase.include?("COMMIT")) && !explain_safe?
+          if (sql.delete_suffix(";").include?(";") || sql.upcase.include?("COMMIT")) && !explain_safe?
             raise ActiveRecord::StatementInvalid, "Unsafe statement"
           end
-          explanation = select_all("EXPLAIN #{sql}").map { |v| v[:"QUERY PLAN"] }.join("\n")
+          explanation = execute("EXPLAIN #{sql}").map { |v| v["QUERY PLAN"] }.join("\n")
         end
 
         explanation
@@ -20,11 +20,12 @@ module PgHero
 
       # TODO rename to explain in 4.0
       # note: this method is not affected by the explain option
-      def explain_v2(sql, analyze: nil, verbose: nil, costs: nil, settings: nil, buffers: nil, wal: nil, timing: nil, summary: nil, format: "text")
+      def explain_v2(sql, analyze: nil, verbose: nil, costs: nil, settings: nil, generic_plan: nil, buffers: nil, wal: nil, timing: nil, summary: nil, format: "text")
         options = []
         add_explain_option(options, "ANALYZE", analyze)
         add_explain_option(options, "VERBOSE", verbose)
         add_explain_option(options, "SETTINGS", settings)
+        add_explain_option(options, "GENERIC_PLAN", generic_plan)
         add_explain_option(options, "COSTS", costs)
         add_explain_option(options, "BUFFERS", buffers)
         add_explain_option(options, "WAL", wal)

data/lib/pghero/methods/indexes.rb

@@ -2,7 +2,7 @@ module PgHero
   module Methods
     module Indexes
       def index_hit_rate
-        select_one <<-SQL
+        select_one <<~SQL
           SELECT
             (sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read), 0) AS rate
           FROM
@@ -11,7 +11,7 @@ module PgHero
       end
 
       def index_caching
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             schemaname AS schema,
             relname AS table,
@@ -29,7 +29,7 @@ module PgHero
       end
 
       def index_usage
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             schemaname AS schema,
             relname AS table,
@@ -47,7 +47,7 @@ module PgHero
       end
 
       def missing_indexes
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             schemaname AS schema,
             relname AS table,
@@ -69,7 +69,7 @@ module PgHero
       end
 
       def unused_indexes(max_scans: 50, across: [])
-        result = select_all_size <<-SQL
+        result = select_all_size <<~SQL
           SELECT
             schemaname AS schema,
             relname AS table,
@@ -104,7 +104,7 @@ module PgHero
       end
 
       def last_stats_reset_time
-        select_one <<-SQL
+        select_one <<~SQL
           SELECT
             pg_stat_get_db_stat_reset_time(oid) AS reset_time
           FROM
@@ -126,7 +126,7 @@ module PgHero
       # TODO parse array properly
       # https://stackoverflow.com/questions/2204058/list-columns-with-indexes-in-postgresql
       def indexes
-        indexes = select_all(<<-SQL
+        indexes = select_all(<<~SQL
           SELECT
             schemaname AS schema,
             t.relname AS table,
@@ -186,7 +186,7 @@ module PgHero
       # thanks @jberkus and @mbanck
       def index_bloat(min_size: nil)
         min_size ||= index_bloat_bytes
-        select_all <<-SQL
+        select_all <<~SQL
           WITH btree_index_atts AS (
             SELECT
               nspname, relname, reltuples, relpages, indrelid, relam,

data/lib/pghero/methods/kill.rb

@@ -11,7 +11,7 @@ module PgHero
       end
 
       def kill_all
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             pg_terminate_backend(pid)
           FROM

data/lib/pghero/methods/maintenance.rb

@@ -9,7 +9,7 @@ module PgHero
         max_value = max_value.to_i
         threshold = threshold.to_i
 
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             n.nspname AS schema,
             c.relname AS table,
@@ -35,7 +35,7 @@ module PgHero
 
       def vacuum_progress
         if server_version_num >= 90600
-          select_all <<-SQL
+          select_all <<~SQL
             SELECT
               pid,
               phase
@@ -50,7 +50,7 @@ module PgHero
       end
 
       def maintenance_info
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             schemaname AS schema,
             relname AS table,

data/lib/pghero/methods/queries.rb

@@ -2,7 +2,7 @@ module PgHero
   module Methods
     module Queries
       def running_queries(min_duration: nil, all: false)
-        query = <<-SQL
+        query = <<~SQL
           SELECT
             pid,
             state,
@@ -36,7 +36,7 @@ module PgHero
       # from https://wiki.postgresql.org/wiki/Lock_Monitoring
       # and https://big-elephants.com/2013-09/exploring-query-locks-in-postgres/
       def blocked_queries
-        query = <<-SQL
+        query = <<~SQL
           SELECT
             COALESCE(blockingl.relation::regclass::text,blockingl.locktype) as locked_item,
             blockeda.pid AS blocked_pid,

data/lib/pghero/methods/query_stats.rb

@@ -162,19 +162,20 @@ module PgHero
         end
       end
 
-      def clean_query_stats
-        PgHero::QueryStats.where(database: id).where("captured_at < ?", 14.days.ago).delete_all
+      def clean_query_stats(before: nil)
+        before ||= 14.days.ago
+        PgHero::QueryStats.where(database: id).where("captured_at < ?", before).delete_all
       end
 
       def slow_queries(query_stats: nil, **options)
-        query_stats ||= self.query_stats(options)
+        query_stats ||= self.query_stats(**options)
         query_stats.select { |q| q[:calls].to_i >= slow_query_calls.to_i && q[:average_time].to_f >= slow_query_ms.to_f }
       end
 
-      def query_hash_stats(query_hash, user: nil)
+      def query_hash_stats(query_hash, user: nil, current: false)
         if historical_query_stats_enabled? && supports_query_hash?
           start_at = 24.hours.ago
-          select_all_stats <<-SQL
+          stats = select_all_stats <<~SQL
             SELECT
               captured_at,
               total_time / 1000 / 60 AS total_minutes,
@@ -191,6 +192,15 @@ module PgHero
             ORDER BY
               1 ASC
           SQL
+          if current
+            captured_at = Time.current
+            current_stats = current_query_stats(query_hash: query_hash, user: user, origin: true)
+            current_stats.each do |r|
+              r[:captured_at] = captured_at
+            end
+            stats += current_stats
+          end
+          stats
         else
           raise NotEnabled, "Query hash stats not enabled"
         end
@@ -199,12 +209,12 @@ module PgHero
       private
 
       # http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
-      def current_query_stats(limit: nil, sort: nil, database: nil, query_hash: nil)
+      def current_query_stats(limit: nil, sort: nil, database: nil, query_hash: nil, user: nil, origin: false)
         if query_stats_enabled?
           limit ||= 100
           sort ||= "total_minutes"
           total_time = server_version_num >= 130000 ? "(total_plan_time + total_exec_time)" : "total_time"
-          query = <<-SQL
+          query = <<~SQL
             WITH query_stats AS (
               SELECT
                 LEFT(query, 10000) AS query,
@@ -223,10 +233,12 @@ module PgHero
                 calls > 0 AND
                 pg_database.datname = #{database ? quote(database) : "current_database()"}
                 #{query_hash ? "AND queryid = #{quote(query_hash)}" : nil}
+                #{user ? "AND rolname = #{quote(user)}" : nil}
             )
             SELECT
               query,
               query AS explainable_query,
+              #{origin ? "(SELECT regexp_matches(query, '.*/\\*(.+?)\\*/'))[1] AS origin," : nil}
               query_hash,
               query_stats.user,
               total_minutes,
@@ -237,7 +249,7 @@ module PgHero
             FROM
               query_stats
             ORDER BY
-              #{quote_table_name(sort)} DESC
+              #{quote_column_name(sort)} DESC
             LIMIT #{limit.to_i}
           SQL
 
@@ -253,7 +265,7 @@ module PgHero
       def historical_query_stats(sort: nil, start_at: nil, end_at: nil, query_hash: nil)
         if historical_query_stats_enabled?
           sort ||= "total_minutes"
-          query = <<-SQL
+          query = <<~SQL
             WITH query_stats AS (
               SELECT
                 #{supports_query_hash? ? "query_hash" : "md5(query)"} AS query_hash,
@@ -286,7 +298,7 @@ module PgHero
             FROM
               query_stats
             ORDER BY
-              #{quote_table_name(sort)} DESC
+              #{quote_column_name(sort)} DESC
             LIMIT 100
           SQL
 
@@ -310,14 +322,14 @@ module PgHero
             all_queries_total_minutes: stats2.sum { |s| s[:all_queries_total_minutes] }
           }
           value[:total_percent] = value[:total_minutes] * 100.0 / value[:all_queries_total_minutes]
-          value[:explainable_query] = stats2.map { |s| s[:explainable_query] }.select { |q| q && explainable?(q) }.first
+          value[:explainable_query] = stats2.map { |s| s[:explainable_query] }.find { |q| q && explainable?(q) }
           query_stats << value
         end
         query_stats
       end
 
       def explainable?(query)
-        query =~ /select/i && !query.include?("?)") && !query.include?("= ?") && !query.include?("$1") && query !~ /limit \?/i
+        query =~ /select/i && (server_version_num >= 160000 || (!query.include?("?)") && !query.include?("= ?") && !query.include?("$1") && query !~ /limit \?/i))
       end
 
       # removes comments
@@ -329,19 +341,17 @@ module PgHero
       def insert_query_stats(db_id, db_query_stats, now)
         values =
           db_query_stats.map do |qs|
-            [
-              db_id,
-              qs[:query],
-              qs[:total_minutes] * 60 * 1000,
-              qs[:calls],
-              now,
-              supports_query_hash? ? qs[:query_hash] : nil,
-              qs[:user]
-            ]
+            {
+              database: db_id,
+              query: qs[:query],
+              total_time: qs[:total_minutes] * 60 * 1000,
+              calls: qs[:calls],
+              captured_at: now,
+              query_hash: supports_query_hash? ? qs[:query_hash] : nil,
+              user: qs[:user]
+            }
           end
-
-        columns = %w[database query total_time calls captured_at query_hash user]
-        insert_stats("pghero_query_stats", columns, values)
+        PgHero::QueryStats.insert_all!(values)
       end
     end
   end

data/lib/pghero/methods/replication.rb

@@ -18,7 +18,7 @@ module PgHero
               "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
             end
 
-          select_one <<-SQL
+          select_one <<~SQL
             SELECT
               CASE
                 WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
@@ -32,7 +32,7 @@ module PgHero
       def replication_slots
         if server_version_num >= 90400
           with_feature_support(:replication_slots, []) do
-            select_all <<-SQL
+            select_all <<~SQL
               SELECT
                 slot_name,
                 database,

data/lib/pghero/methods/sequences.rb

@@ -7,7 +7,7 @@ module PgHero
         # it's what information_schema.columns uses
         # also, exclude temporary tables to prevent error
         # when accessing across sessions
-        sequences = select_all <<-SQL
+        sequences = select_all <<~SQL
           SELECT
             n.nspname AS table_schema,
             c.relname AS table,
@@ -39,14 +39,19 @@ module PgHero
 
         add_sequence_attributes(sequences)
 
-        sequences.select { |s| s[:readable] }.each_slice(1024) do |slice|
-          sql = slice.map { |s| "SELECT last_value FROM #{quote_ident(s[:schema])}.#{quote_ident(s[:sequence])}" }.join(" UNION ALL ")
+        last_value = {}
+        sequences.select { |s| s[:readable] }.map { |s| [s[:schema], s[:sequence]] }.uniq.each_slice(1024) do |slice|
+          sql = slice.map { |s| "SELECT last_value FROM #{quote_ident(s[0])}.#{quote_ident(s[1])}" }.join(" UNION ALL ")
 
           select_all(sql).zip(slice) do |row, seq|
-            seq[:last_value] = row[:last_value]
+            last_value[seq] = row[:last_value]
           end
         end
 
+        sequences.select { |s| s[:readable] }.each do |seq|
+          seq[:last_value] = last_value[[seq[:schema], seq[:sequence]]]
+        end
+
         # use to_s for unparsable sequences
         sequences.sort_by { |s| s[:sequence].to_s }
       end
@@ -84,7 +89,7 @@ module PgHero
       # also adds schema if missing
       def add_sequence_attributes(sequences)
         # fetch data
-        sequence_attributes = select_all <<-SQL
+        sequence_attributes = select_all <<~SQL
           SELECT
             n.nspname AS schema,
             c.relname AS sequence,

data/lib/pghero/methods/settings.rb

@@ -3,7 +3,14 @@ module PgHero
     module Settings
       def settings
         names =
-          if server_version_num >= 90500
+          if server_version_num >= 100000
+            %i(
+              max_connections shared_buffers effective_cache_size maintenance_work_mem
+              checkpoint_completion_target wal_buffers default_statistics_target
+              random_page_cost effective_io_concurrency work_mem huge_pages
+              min_wal_size max_wal_size
+            )
+          elsif server_version_num >= 90500
             %i(
               max_connections shared_buffers effective_cache_size work_mem
               maintenance_work_mem min_wal_size max_wal_size checkpoint_completion_target

data/lib/pghero/methods/space.rb

@@ -6,11 +6,11 @@ module PgHero
       end
 
       def relation_sizes
-        select_all_size <<-SQL
+        select_all_size <<~SQL
           SELECT
             n.nspname AS schema,
             c.relname AS relation,
-            CASE WHEN c.relkind = 'r' THEN 'table' ELSE 'index' END AS type,
+            CASE c.relkind WHEN 'r' THEN 'table' WHEN 'm' then 'matview' ELSE 'index' END AS type,
             pg_table_size(c.oid) AS size_bytes
           FROM
             pg_class c
@@ -19,7 +19,7 @@ module PgHero
           WHERE
             n.nspname NOT IN ('pg_catalog', 'information_schema')
             AND n.nspname !~ '^pg_toast'
-            AND c.relkind IN ('r', 'i')
+            AND c.relkind IN ('r', 'm', 'i')
           ORDER BY
             pg_table_size(c.oid) DESC,
             2 ASC
@@ -27,7 +27,7 @@ module PgHero
       end
 
       def table_sizes
-        select_all_size <<-SQL
+        select_all_size <<~SQL
           SELECT
             n.nspname AS schema,
             c.relname AS table,
@@ -52,7 +52,7 @@ module PgHero
           sizes = relation_sizes.to_h { |r| [[r[:schema], r[:relation]], r[:size_bytes]] }
           start_at = days.days.ago
 
-          stats = select_all_stats <<-SQL
+          stats = select_all_stats <<~SQL
             WITH t AS (
               SELECT
                 schema,
@@ -95,7 +95,7 @@ module PgHero
           sizes = relation_sizes.map { |r| [[r[:schema], r[:relation]], r[:size_bytes]] }.to_h
           start_at = 30.days.ago
 
-          stats = select_all_stats <<-SQL
+          stats = select_all_stats <<~SQL
             SELECT
               captured_at,
               size AS size_bytes
@@ -121,16 +121,22 @@ module PgHero
 
       def capture_space_stats
         now = Time.now
-        columns = %w(database schema relation size captured_at)
-        values = []
-        relation_sizes.each do |rs|
-          values << [id, rs[:schema], rs[:relation], rs[:size_bytes].to_i, now]
-        end
-        insert_stats("pghero_space_stats", columns, values) if values.any?
+        values =
+          relation_sizes.map do |rs|
+            {
+              database: id,
+              schema: rs[:schema],
+              relation: rs[:relation],
+              size: rs[:size_bytes].to_i,
+              captured_at: now
+            }
+          end
+        PgHero::SpaceStats.insert_all!(values) if values.any?
       end
 
-      def clean_space_stats
-        PgHero::SpaceStats.where(database: id).where("captured_at < ?", 90.days.ago).delete_all
+      def clean_space_stats(before: nil)
+        before ||= 90.days.ago
+        PgHero::SpaceStats.where(database: id).where("captured_at < ?", before).delete_all
       end
 
       def space_stats_enabled?

data/lib/pghero/methods/suggested_indexes.rb

@@ -46,7 +46,7 @@ module PgHero
                   indexes += existing_columns["brin"][index[:table]]
                 end
 
-                covering_index = indexes.find { |e| index_covers?(e.map { |v| v.sub(/ inet_ops\z/, "") }, index[:columns]) }
+                covering_index = indexes.find { |e| index_covers?(e.map { |v| v.delete_suffix(" inet_ops") }, index[:columns]) }
                 if covering_index
                   best_index[:covering_index] = covering_index
                   best_index[:explanation] = "Covered by index on (#{covering_index.join(", ")})"
@@ -79,7 +79,9 @@ module PgHero
         suggested_indexes.each do |index|
           p index
           if create
-            connection.execute("CREATE INDEX CONCURRENTLY ON #{quote_table_name(index[:table])} (#{index[:columns].map { |c| quote_table_name(c) }.join(",")})")
+            with_connection do |connection|
+              connection.execute("CREATE INDEX CONCURRENTLY ON #{quote_table_name(index[:table])} (#{index[:columns].map { |c| quote_column_name(c) }.join(",")})")
+            end
           end
         end
       end
@@ -194,6 +196,7 @@ module PgHero
       end
 
       def best_index_structure(statement)
+        return {error: "Empty statement"} if statement.to_s.empty?
         return {error: "Too large"} if statement.to_s.length > 10000
 
         begin
@@ -286,27 +289,31 @@ module PgHero
           else
             raise "Not Implemented"
           end
-        elsif aexpr && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(aexpr.name.first.string.str)
-          [{column: aexpr.lexpr.column_ref.fields.last.string.str, op: aexpr.name.first.string.str}]
+        elsif aexpr && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(aexpr.name.first.string.send(str_method))
+          [{column: aexpr.lexpr.column_ref.fields.last.string.send(str_method), op: aexpr.name.first.string.send(str_method)}]
         elsif tree.null_test
           op = tree.null_test.nulltesttype == :IS_NOT_NULL ? "not_null" : "null"
-          [{column: tree.null_test.arg.column_ref.fields.last.string.str, op: op}]
+          [{column: tree.null_test.arg.column_ref.fields.last.string.send(str_method), op: op}]
         else
           raise "Not Implemented"
         end
       end
 
+      def str_method
+        @str_method ||= Gem::Version.new(PgQuery::VERSION) >= Gem::Version.new("4") ? :sval : :str
+      end
+
       def parse_sort(sort_clause)
         sort_clause.map do |v|
           {
-            column: v.sort_by.node.column_ref.fields.last.string.str,
+            column: v.sort_by.node.column_ref.fields.last.string.send(str_method),
             direction: v.sort_by.sortby_dir == :SORTBY_DESC ? "desc" : "asc"
           }
         end
       end
 
       def column_stats(schema: nil, table: nil)
-        select_all <<-SQL
+        select_all <<~SQL
           SELECT
             schemaname AS schema,
             tablename AS table,