pghero_fork 2.7.3

data/lib/pghero/engine.rb

@@ -0,0 +1,16 @@
+module PgHero
+  class Engine < ::Rails::Engine
+    isolate_namespace PgHero
+
+    initializer "pghero", group: :all do |config|
+      PgHero.time_zone = PgHero.config["time_zone"] if PgHero.config["time_zone"]
+      # View files working with flash
+      config.middleware.use ActionDispatch::Flash
+
+      # We can add all of the public assets from our engine and make them
+      # available to use.  This allows us to use javascripts, images, stylesheets
+      # etc.
+      config.middleware.insert_before(::ActionDispatch::Flash, ::ActionDispatch::Static, "#{root}/public")
+    end
+  end
+end

data/lib/pghero/methods/basic.rb

@@ -0,0 +1,160 @@
+module PgHero
+  module Methods
+    module Basic
+      def ssl_used?
+        ssl_used = nil
+        with_transaction(rollback: true) do
+          begin
+            execute("CREATE EXTENSION IF NOT EXISTS sslinfo")
+          rescue ActiveRecord::StatementInvalid
+            # not superuser
+          end
+          ssl_used = select_one("SELECT ssl_is_used()")
+        end
+        ssl_used
+      end
+
+      def database_name
+        select_one("SELECT current_database()")
+      end
+
+      def current_user
+        select_one("SELECT current_user")
+      end
+
+      def server_version
+        @server_version ||= select_one("SHOW server_version")
+      end
+
+      def server_version_num
+        @server_version_num ||= select_one("SHOW server_version_num").to_i
+      end
+
+      def quote_ident(value)
+        connection.quote_column_name(value)
+      end
+
+      private
+
+      def select_all(sql, conn: nil, query_columns: [])
+        conn ||= connection
+        # squish for logs
+        retries = 0
+        begin
+          result = conn.select_all(add_source(squish(sql)))
+          if ActiveRecord::VERSION::STRING.to_f >= 6.1
+            result = result.map(&:symbolize_keys)
+          else
+            result = result.map { |row| Hash[row.map { |col, val| [col.to_sym, result.column_types[col].send(:cast_value, val)] }] }
+          end
+          if filter_data
+            query_columns.each do |column|
+              result.each do |row|
+                begin
+                  row[column] = PgQuery.normalize(row[column])
+                rescue PgQuery::ParseError
+                  # try replacing "interval $1" with "$1::interval"
+                  # see https://github.com/lfittl/pg_query/issues/169 for more info
+                  # this is not ideal since it changes the query slightly
+                  # we could skip normalization
+                  # but this has a very small chance of data leakage
+                  begin
+                    row[column] = PgQuery.normalize(row[column].gsub(/\binterval\s+(\$\d+)\b/i, "\\1::interval"))
+                  rescue PgQuery::ParseError
+                    row[column] = "<unable to filter data>"
+                  end
+                end
+              end
+            end
+          end
+          result
+        rescue ActiveRecord::StatementInvalid => e
+          # fix for random internal errors
+          if e.message.include?("PG::InternalError") && retries < 2
+            retries += 1
+            sleep(0.1)
+            retry
+          else
+            raise e
+          end
+        end
+      end
+
+      def select_all_stats(sql, **options)
+        select_all(sql, **options, conn: stats_connection)
+      end
+
+      def select_all_size(sql)
+        result = select_all(sql)
+        result.each do |row|
+          row[:size] = PgHero.pretty_size(row[:size_bytes])
+        end
+        result
+      end
+
+      def select_one(sql, conn: nil)
+        select_all(sql, conn: conn).first.values.first
+      end
+
+      def select_one_stats(sql)
+        select_one(sql, conn: stats_connection)
+      end
+
+      def execute(sql)
+        connection.execute(add_source(sql))
+      end
+
+      def connection
+        connection_model.connection
+      end
+
+      def stats_connection
+        ::PgHero::Stats.connection
+      end
+
+      def insert_stats(table, columns, values)
+        values = values.map { |v| "(#{v.map { |v2| quote(v2) }.join(",")})" }.join(",")
+        columns = columns.map { |v| quote_table_name(v) }.join(",")
+        stats_connection.execute("INSERT INTO #{quote_table_name(table)} (#{columns}) VALUES #{values}")
+      end
+
+      # from ActiveSupport
+      def squish(str)
+        str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
+      end
+
+      def add_source(sql)
+        "#{sql} /*pghero*/"
+      end
+
+      def quote(value)
+        connection.quote(value)
+      end
+
+      def quote_table_name(value)
+        connection.quote_table_name(value)
+      end
+
+      def unquote(part)
+        if part && part.start_with?('"')
+          part[1..-2]
+        else
+          part
+        end
+      end
+
+      def with_transaction(lock_timeout: nil, statement_timeout: nil, rollback: false)
+        connection_model.transaction do
+          select_all "SET LOCAL statement_timeout = #{statement_timeout.to_i}" if statement_timeout
+          select_all "SET LOCAL lock_timeout = #{lock_timeout.to_i}" if lock_timeout
+          yield
+          raise ActiveRecord::Rollback if rollback
+        end
+      end
+
+      def table_exists?(table)
+        stats_connection.table_exists?(table)
+      end
+    end
+  end
+end

data/lib/pghero/methods/connections.rb

@@ -0,0 +1,77 @@
+module PgHero
+  module Methods
+    module Connections
+      def connections
+        if server_version_num >= 90500
+          select_all <<-SQL
+            SELECT
+              pg_stat_activity.pid,
+              datname AS database,
+              usename AS user,
+              application_name AS source,
+              client_addr AS ip,
+              state,
+              ssl
+            FROM
+              pg_stat_activity
+            LEFT JOIN
+              pg_stat_ssl ON pg_stat_activity.pid = pg_stat_ssl.pid
+            ORDER BY
+              pg_stat_activity.pid
+          SQL
+        else
+          select_all <<-SQL
+            SELECT
+              pid,
+              datname AS database,
+              usename AS user,
+              application_name AS source,
+              client_addr AS ip,
+              state
+            FROM
+              pg_stat_activity
+            ORDER BY
+              pid
+          SQL
+        end
+      end
+
+      def total_connections
+        select_one("SELECT COUNT(*) FROM pg_stat_activity")
+      end
+
+      def connection_states
+        states = select_all <<-SQL
+          SELECT
+            state,
+            COUNT(*) AS connections
+          FROM
+            pg_stat_activity
+          GROUP BY
+            1
+          ORDER BY
+            2 DESC, 1
+        SQL
+
+        Hash[states.map { |s| [s[:state], s[:connections]] }]
+      end
+
+      def connection_sources
+        select_all <<-SQL
+          SELECT
+            datname AS database,
+            usename AS user,
+            application_name AS source,
+            client_addr AS ip,
+            COUNT(*) AS total_connections
+          FROM
+            pg_stat_activity
+          GROUP BY
+            1, 2, 3, 4
+          ORDER BY
+            5 DESC, 1, 2, 3, 4
+        SQL
+      end
+    end
+  end
+end

data/lib/pghero/methods/constraints.rb

@@ -0,0 +1,30 @@
+module PgHero
+  module Methods
+    module Constraints
+      # referenced fields can be nil
+      # as not all constraints are foreign keys
+      def invalid_constraints
+        select_all <<-SQL
+          SELECT
+            nsp.nspname AS schema,
+            rel.relname AS table,
+            con.conname AS name,
+            fnsp.nspname AS referenced_schema,
+            frel.relname AS referenced_table
+          FROM
+            pg_catalog.pg_constraint con
+          INNER JOIN
+            pg_catalog.pg_class rel ON rel.oid = con.conrelid
+          LEFT JOIN
+            pg_catalog.pg_class frel ON frel.oid = con.confrelid
+          LEFT JOIN
+            pg_catalog.pg_namespace nsp ON nsp.oid = con.connamespace
+          LEFT JOIN
+            pg_catalog.pg_namespace fnsp ON fnsp.oid = frel.relnamespace
+          WHERE
+            con.convalidated = 'f'
+        SQL
+      end
+    end
+  end
+end

data/lib/pghero/methods/explain.rb

@@ -0,0 +1,29 @@
+module PgHero
+  module Methods
+    module Explain
+      def explain(sql)
+        sql = squish(sql)
+        explanation = nil
+
+        # use transaction for safety
+        with_transaction(statement_timeout: (explain_timeout_sec * 1000).round, rollback: true) do
+          if (sql.sub(/;\z/, "").include?(";") || sql.upcase.include?("COMMIT")) && !explain_safe?
+            raise ActiveRecord::StatementInvalid, "Unsafe statement"
+          end
+          explanation = select_all("EXPLAIN #{sql}").map { |v| v[:"QUERY PLAN"] }.join("\n")
+        end
+
+        explanation
+      end
+
+      private
+
+      def explain_safe?
+        select_all("SELECT 1; SELECT 1")
+        false
+      rescue ActiveRecord::StatementInvalid
+        true
+      end
+    end
+  end
+end

data/lib/pghero/methods/indexes.rb

@@ -0,0 +1,332 @@
+module PgHero
+  module Methods
+    module Indexes
+      def index_hit_rate
+        select_one <<-SQL
+          SELECT
+            (sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read), 0) AS rate
+          FROM
+            pg_statio_user_indexes
+        SQL
+      end
+
+      def index_caching
+        select_all <<-SQL
+          SELECT
+            schemaname AS schema,
+            relname AS table,
+            indexrelname AS index,
+            CASE WHEN idx_blks_hit + idx_blks_read = 0 THEN
+              0
+            ELSE
+              ROUND(1.0 * idx_blks_hit / (idx_blks_hit + idx_blks_read), 2)
+            END AS hit_rate
+          FROM
+            pg_statio_user_indexes
+          ORDER BY
+            3 DESC, 1
+        SQL
+      end
+
+      def index_usage
+        select_all <<-SQL
+          SELECT
+            schemaname AS schema,
+            relname AS table,
+            CASE idx_scan
+              WHEN 0 THEN 'Insufficient data'
+              ELSE (100 * idx_scan / (seq_scan + idx_scan))::text
+            END percent_of_times_index_used,
+            n_live_tup AS estimated_rows
+          FROM
+            pg_stat_user_tables
+          ORDER BY
+            n_live_tup DESC,
+            relname ASC
+         SQL
+      end
+
+      def missing_indexes
+        select_all <<-SQL
+          SELECT
+            schemaname AS schema,
+            relname AS table,
+            CASE idx_scan
+              WHEN 0 THEN 'Insufficient data'
+              ELSE (100 * idx_scan / (seq_scan + idx_scan))::text
+            END percent_of_times_index_used,
+            n_live_tup AS estimated_rows
+          FROM
+            pg_stat_user_tables
+          WHERE
+            idx_scan > 0
+            AND (100 * idx_scan / (seq_scan + idx_scan)) < 95
+            AND n_live_tup >= 10000
+          ORDER BY
+            n_live_tup DESC,
+            relname ASC
+         SQL
+      end
+
+      def unused_indexes(max_scans: 50, across: [])
+        result = select_all_size <<-SQL
+          SELECT
+            schemaname AS schema,
+            relname AS table,
+            indexrelname AS index,
+            pg_relation_size(i.indexrelid) AS size_bytes,
+            idx_scan as index_scans
+          FROM
+            pg_stat_user_indexes ui
+          INNER JOIN
+            pg_index i ON ui.indexrelid = i.indexrelid
+          WHERE
+            NOT indisunique
+            AND idx_scan <= #{max_scans.to_i}
+          ORDER BY
+            pg_relation_size(i.indexrelid) DESC,
+            relname ASC
+        SQL
+
+        across.each do |database_id|
+          database = PgHero.databases.values.find { |d| d.id == database_id }
+          raise PgHero::Error, "Database not found: #{database_id}" unless database
+          across_result = Set.new(database.unused_indexes(max_scans: max_scans).map { |v| [v[:schema], v[:index]] })
+          result.select! { |v| across_result.include?([v[:schema], v[:index]]) }
+        end
+
+        result
+      end
+
+      def reset_stats
+        execute("SELECT pg_stat_reset()")
+        true
+      end
+
+      def last_stats_reset_time
+        select_one <<-SQL
+          SELECT
+            pg_stat_get_db_stat_reset_time(oid) AS reset_time
+          FROM
+            pg_database
+          WHERE
+            datname = current_database()
+        SQL
+      end
+
+      def invalid_indexes(indexes: nil)
+        indexes = (indexes || self.indexes).select { |i| !i[:valid] && !i[:creating] }
+        indexes.each do |index|
+          # map name -> index for backward compatibility
+          index[:index] = index[:name]
+        end
+        indexes
+      end
+
+      # TODO parse array properly
+      # https://stackoverflow.com/questions/2204058/list-columns-with-indexes-in-postgresql
+      def indexes
+        indexes = select_all(<<-SQL
+          SELECT
+            schemaname AS schema,
+            t.relname AS table,
+            ix.relname AS name,
+            regexp_replace(pg_get_indexdef(i.indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
+            regexp_replace(pg_get_indexdef(i.indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
+            indisunique AS unique,
+            indisprimary AS primary,
+            indisvalid AS valid,
+            indexprs::text,
+            indpred::text,
+            pg_get_indexdef(i.indexrelid) AS definition
+          FROM
+            pg_index i
+          INNER JOIN
+            pg_class t ON t.oid = i.indrelid
+          INNER JOIN
+            pg_class ix ON ix.oid = i.indexrelid
+          LEFT JOIN
+            pg_stat_user_indexes ui ON ui.indexrelid = i.indexrelid
+          WHERE
+            schemaname IS NOT NULL
+          ORDER BY
+            1, 2
+        SQL
+        ).map { |v| v[:columns] = v[:columns].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
+
+        # determine if any invalid indexes being created
+        # hacky, but works for simple cases
+        # can be a race condition, but that's fine
+        invalid_indexes = indexes.select { |i| !i[:valid] }
+        if invalid_indexes.any?
+          create_index_queries = running_queries.select { |q| /\s*CREATE\s+INDEX\s+CONCURRENTLY\s+/i.match(q[:query]) }
+          invalid_indexes.each do |index|
+            index[:creating] = create_index_queries.any? { |q| q[:query].include?(index[:table]) && index[:columns].all? { |c| q[:query].include?(c) } }
+          end
+        end
+
+        indexes
+      end
+
+      def duplicate_indexes(indexes: nil)
+        dup_indexes = []
+
+        indexes_by_table = (indexes || self.indexes).group_by { |i| [i[:schema], i[:table]] }
+        indexes_by_table.values.flatten.select { |i| i[:valid] && !i[:primary] && !i[:unique] }.each do |index|
+          covering_index = indexes_by_table[[index[:schema], index[:table]]].find { |i| i[:valid] && i[:name] != index[:name] && index_covers?(i[:columns], index[:columns]) && i[:using] == index[:using] && i[:indexprs] == index[:indexprs] && i[:indpred] == index[:indpred] }
+          if covering_index && (covering_index[:columns] != index[:columns] || index[:name] > covering_index[:name] || covering_index[:primary] || covering_index[:unique])
+            dup_indexes << {unneeded_index: index, covering_index: covering_index}
+          end
+        end
+
+        dup_indexes.sort_by { |i| ui = i[:unneeded_index]; [ui[:table], ui[:columns]] }
+      end
+
+      # https://gist.github.com/mbanck/9976015/71888a24e464e2f772182a7eb54f15a125edf398
+      # thanks @jberkus and @mbanck
+      def index_bloat(min_size: nil)
+        min_size ||= index_bloat_bytes
+        select_all <<-SQL
+          WITH btree_index_atts AS (
+            SELECT
+              nspname, relname, reltuples, relpages, indrelid, relam,
+              regexp_split_to_table(indkey::text, ' ')::smallint AS attnum,
+              indexrelid as index_oid
+            FROM
+              pg_index
+            JOIN
+              pg_class ON pg_class.oid = pg_index.indexrelid
+            JOIN
+              pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+            JOIN
+              pg_am ON pg_class.relam = pg_am.oid
+            WHERE
+              pg_am.amname = 'btree'
+          ),
+          index_item_sizes AS (
+            SELECT
+              i.nspname,
+              i.relname,
+              i.reltuples,
+              i.relpages,
+              i.relam,
+              (quote_ident(s.schemaname) || '.' || quote_ident(s.tablename))::regclass AS starelid,
+              a.attrelid AS table_oid, index_oid,
+              current_setting('block_size')::numeric AS bs,
+              /* MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) */
+              CASE
+                WHEN version() ~ 'mingw32' OR version() ~ '64-bit' THEN 8
+                ELSE 4
+              END AS maxalign,
+              24 AS pagehdr,
+              /* per tuple header: add index_attribute_bm if some cols are null-able */
+              CASE WHEN max(coalesce(s.null_frac,0)) = 0
+                THEN 2
+                ELSE 6
+              END AS index_tuple_hdr,
+              /* data len: we remove null values save space using it fractionnal part from stats */
+              sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 2048) ) AS nulldatawidth
+            FROM
+              pg_attribute AS a
+            JOIN
+              pg_stats AS s ON (quote_ident(s.schemaname) || '.' || quote_ident(s.tablename))::regclass=a.attrelid AND s.attname = a.attname
+            JOIN
+              btree_index_atts AS i ON i.indrelid = a.attrelid AND a.attnum = i.attnum
+            WHERE
+              a.attnum > 0
+            GROUP BY
+              1, 2, 3, 4, 5, 6, 7, 8, 9
+          ),
+          index_aligned AS (
+            SELECT
+              maxalign,
+              bs,
+              nspname,
+              relname AS index_name,
+              reltuples,
+              relpages,
+              relam,
+              table_oid,
+              index_oid,
+              ( 2 +
+                maxalign - CASE /* Add padding to the index tuple header to align on MAXALIGN */
+                  WHEN index_tuple_hdr%maxalign = 0 THEN maxalign
+                  ELSE index_tuple_hdr%maxalign
+                END
+              + nulldatawidth + maxalign - CASE /* Add padding to the data to align on MAXALIGN */
+                  WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
+                  ELSE nulldatawidth::integer%maxalign
+                END
+              )::numeric AS nulldatahdrwidth, pagehdr
+            FROM
+              index_item_sizes AS s1
+          ),
+          otta_calc AS (
+            SELECT
+              bs,
+              nspname,
+              table_oid,
+              index_oid,
+              index_name,
+              relpages,
+              coalesce(
+                ceil((reltuples*(4+nulldatahdrwidth))/(bs-pagehdr::float)) +
+                CASE WHEN am.amname IN ('hash','btree') THEN 1 ELSE 0 END , 0 /* btree and hash have a metadata reserved block */
+              ) AS otta
+            FROM
+              index_aligned AS s2
+            LEFT JOIN
+              pg_am am ON s2.relam = am.oid
+          ),
+          raw_bloat AS (
+            SELECT
+              nspname,
+              c.relname AS table_name,
+              index_name,
+              bs*(sub.relpages)::bigint AS totalbytes,
+              CASE
+                WHEN sub.relpages <= otta THEN 0
+                ELSE bs*(sub.relpages-otta)::bigint END
+                AS wastedbytes,
+              CASE
+                WHEN sub.relpages <= otta
+                THEN 0 ELSE bs*(sub.relpages-otta)::bigint * 100 / (bs*(sub.relpages)::bigint) END
+                AS realbloat,
+              pg_relation_size(sub.table_oid) as table_bytes,
+              stat.idx_scan as index_scans,
+              stat.indexrelid
+            FROM
+              otta_calc AS sub
+            JOIN
+              pg_class AS c ON c.oid=sub.table_oid
+            JOIN
+              pg_stat_user_indexes AS stat ON sub.index_oid = stat.indexrelid
+          )
+          SELECT
+            nspname AS schema,
+            table_name AS table,
+            index_name AS index,
+            wastedbytes AS bloat_bytes,
+            totalbytes AS index_bytes,
+            pg_get_indexdef(rb.indexrelid) AS definition,
+            indisprimary AS primary
+          FROM
+            raw_bloat rb
+          INNER JOIN
+            pg_index i ON i.indexrelid = rb.indexrelid
+          WHERE
+            wastedbytes >= #{min_size.to_i}
+          ORDER BY
+            wastedbytes DESC,
+            index_name
+        SQL
+      end
+
+      protected
+
+      def index_covers?(indexed_columns, columns)
+        indexed_columns.first(columns.size) == columns
+      end
+    end
+  end
+end