pghero 1.5.3 → 2.0.0

data/lib/pghero/methods/connections.rb

@@ -2,64 +2,24 @@ module PgHero
   module Methods
     module Connections
       def total_connections
-        select_all("SELECT COUNT(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid()").first["count"].to_i
+        select_one("SELECT COUNT(*) FROM pg_stat_activity")
       end
 
-      def connection_sources(options = {})
-        if options[:by_database_and_user]
-          select_all <<-SQL
-            SELECT
-              datname AS database,
-              usename AS user,
-              application_name AS source,
-              client_addr AS ip,
-              COUNT(*) AS total_connections
-            FROM
-              pg_stat_activity
-            WHERE
-              pid <> pg_backend_pid()
-            GROUP BY
-              1, 2, 3, 4
-            ORDER BY
-              5 DESC, 1, 2, 3, 4
-          SQL
-        elsif options[:by_database]
-          select_all <<-SQL
-            SELECT
-              application_name AS source,
-              client_addr AS ip,
-              datname AS database,
-              COUNT(*) AS total_connections
-            FROM
-              pg_stat_activity
-            WHERE
-              pid <> pg_backend_pid()
-            GROUP BY
-              1, 2, 3
-            ORDER BY
-              COUNT(*) DESC,
-              application_name ASC,
-              client_addr ASC
-          SQL
-        else
-          select_all <<-SQL
-            SELECT
-              application_name AS source,
-              client_addr AS ip,
-              COUNT(*) AS total_connections
-            FROM
-              pg_stat_activity
-            WHERE
-              pid <> pg_backend_pid()
-            GROUP BY
-              application_name,
-              ip
-            ORDER BY
-              COUNT(*) DESC,
-              application_name ASC,
-              client_addr ASC
-          SQL
-        end
+      def connection_sources
+        select_all <<-SQL
+          SELECT
+            datname AS database,
+            usename AS user,
+            application_name AS source,
+            client_addr AS ip,
+            COUNT(*) AS total_connections
+          FROM
+            pg_stat_activity
+          GROUP BY
+            1, 2, 3, 4
+          ORDER BY
+            5 DESC, 1, 2, 3, 4
+        SQL
       end
     end
   end

data/lib/pghero/methods/explain.rb

@@ -4,15 +4,13 @@ module PgHero
       def explain(sql)
         sql = squish(sql)
         explanation = nil
-        explain_safe = explain_safe?
 
         # use transaction for safety
-        connection_model.transaction do
-          if !explain_safe && (sql.sub(/;\z/, "").include?(";") || sql.upcase.include?("COMMIT"))
+        with_transaction(statement_timeout: 10000, rollback: true) do
+          if (sql.sub(/;\z/, "").include?(";") || sql.upcase.include?("COMMIT")) && !explain_safe?
             raise ActiveRecord::StatementInvalid, "Unsafe statement"
           end
-          explanation = select_all("EXPLAIN #{sql}").map { |v| v["QUERY PLAN"] }.join("\n")
-          raise ActiveRecord::Rollback
+          explanation = select_all("EXPLAIN #{sql}").map { |v| v[:"QUERY PLAN"] }.join("\n")
         end
 
         explanation

data/lib/pghero/methods/indexes.rb

@@ -2,20 +2,20 @@ module PgHero
   module Methods
     module Indexes
       def index_hit_rate
-        select_all(<<-SQL
+        select_one <<-SQL
           SELECT
             (sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read), 0) AS rate
           FROM
             pg_statio_user_indexes
         SQL
-        ).first["rate"].to_f
       end
 
       def index_caching
         select_all <<-SQL
           SELECT
-            indexrelname AS index,
+            schemaname AS schema,
             relname AS table,
+            indexrelname AS index,
             CASE WHEN idx_blks_hit + idx_blks_read = 0 THEN
               0
             ELSE
@@ -37,7 +37,7 @@ module PgHero
               WHEN 0 THEN 'Insufficient data'
               ELSE (100 * idx_scan / (seq_scan + idx_scan))::text
             END percent_of_times_index_used,
-            n_live_tup rows_in_table
+            n_live_tup AS estimated_rows
           FROM
             pg_stat_user_tables
           ORDER BY
@@ -55,7 +55,7 @@ module PgHero
               WHEN 0 THEN 'Insufficient data'
               ELSE (100 * idx_scan / (seq_scan + idx_scan))::text
             END percent_of_times_index_used,
-            n_live_tup rows_in_table
+            n_live_tup AS estimated_rows
           FROM
             pg_stat_user_tables
           WHERE
@@ -68,13 +68,13 @@ module PgHero
          SQL
       end
 
-      def unused_indexes
-        select_all <<-SQL
+      def unused_indexes(max_scans: 50)
+        select_all_size <<-SQL
           SELECT
             schemaname AS schema,
             relname AS table,
             indexrelname AS index,
-            pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size,
+            pg_relation_size(i.indexrelid) AS size_bytes,
             idx_scan as index_scans
           FROM
             pg_stat_user_indexes ui
@@ -82,13 +82,29 @@ module PgHero
             pg_index i ON ui.indexrelid = i.indexrelid
           WHERE
             NOT indisunique
-            AND idx_scan < 50
+            AND idx_scan <= #{max_scans.to_i}
           ORDER BY
             pg_relation_size(i.indexrelid) DESC,
             relname ASC
         SQL
       end
 
+      def reset_stats
+        execute("SELECT pg_stat_reset()")
+        true
+      end
+
+      def last_stats_reset_time
+        select_one <<-SQL
+          SELECT
+            pg_stat_get_db_stat_reset_time(oid) AS reset_time
+          FROM
+            pg_database
+          WHERE
+            datname = current_database()
+        SQL
+      end
+
       def invalid_indexes
         select_all <<-SQL
           SELECT
@@ -114,40 +130,182 @@ module PgHero
       def indexes
         select_all(<<-SQL
           SELECT
+            schemaname AS schema,
             t.relname AS table,
             ix.relname AS name,
-            regexp_replace(pg_get_indexdef(indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
-            regexp_replace(pg_get_indexdef(indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
+            regexp_replace(pg_get_indexdef(i.indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
+            regexp_replace(pg_get_indexdef(i.indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
             indisunique AS unique,
             indisprimary AS primary,
             indisvalid AS valid,
             indexprs::text,
             indpred::text,
-            pg_get_indexdef(indexrelid) AS definition
+            pg_get_indexdef(i.indexrelid) AS definition
           FROM
             pg_index i
           INNER JOIN
             pg_class t ON t.oid = i.indrelid
           INNER JOIN
             pg_class ix ON ix.oid = i.indexrelid
+          LEFT JOIN
+            pg_stat_user_indexes ui ON ui.indexrelid = i.indexrelid
+          WHERE
+            schemaname IS NOT NULL
           ORDER BY
             1, 2
         SQL
-        ).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
+        ).map { |v| v[:columns] = v[:columns].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
       end
 
-      def duplicate_indexes
-        indexes = []
+      def duplicate_indexes(indexes: nil)
+        dup_indexes = []
 
-        indexes_by_table = self.indexes.group_by { |i| i["table"] }
-        indexes_by_table.values.flatten.select { |i| PgHero.falsey?(i["primary"]) && PgHero.falsey?(i["unique"]) && !i["indexprs"] && !i["indpred"] && PgHero.truthy?(i["valid"]) }.each do |index|
-          covering_index = indexes_by_table[index["table"]].find { |i| index_covers?(i["columns"], index["columns"]) && i["using"] == index["using"] && i["name"] != index["name"] && !i["indexprs"] && !i["indpred"] && PgHero.truthy?(i["valid"]) }
-          if covering_index
-            indexes << {"unneeded_index" => index, "covering_index" => covering_index}
+        indexes_by_table = (indexes || self.indexes).group_by { |i| i[:table] }
+        indexes_by_table.values.flatten.select { |i| !i[:primary] && !i[:unique] && !i[:indexprs] && !i[:indpred] && i[:valid] }.each do |index|
+          covering_index = indexes_by_table[index[:table]].find { |i| index_covers?(i[:columns], index[:columns]) && i[:using] == index[:using] && i[:name] != index[:name] && i[:schema] == index[:schema] && !i[:indexprs] && !i[:indpred] && i[:valid] }
+          if covering_index && (covering_index[:columns] != index[:columns] || index[:name] > covering_index[:name])
+            dup_indexes << {unneeded_index: index, covering_index: covering_index}
           end
         end
 
-        indexes.sort_by { |i| ui = i["unneeded_index"]; [ui["table"], ui["columns"]] }
+        dup_indexes.sort_by { |i| ui = i[:unneeded_index]; [ui[:table], ui[:columns]] }
+      end
+
+      # https://gist.github.com/mbanck/9976015/71888a24e464e2f772182a7eb54f15a125edf398
+      # thanks @jberkus and @mbanck
+      def index_bloat(min_size: nil)
+        min_size ||= index_bloat_bytes
+        select_all <<-SQL
+          WITH btree_index_atts AS (
+            SELECT
+              nspname, relname, reltuples, relpages, indrelid, relam,
+              regexp_split_to_table(indkey::text, ' ')::smallint AS attnum,
+              indexrelid as index_oid
+            FROM
+              pg_index
+            JOIN
+              pg_class ON pg_class.oid=pg_index.indexrelid
+            JOIN
+              pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+            JOIN
+              pg_am ON pg_class.relam = pg_am.oid
+            WHERE
+              pg_am.amname = 'btree'
+          ),
+          index_item_sizes AS (
+            SELECT
+              i.nspname,
+              i.relname,
+              i.reltuples,
+              i.relpages,
+              i.relam,
+              (quote_ident(s.schemaname) || '.' || quote_ident(s.tablename))::regclass AS starelid,
+              a.attrelid AS table_oid, index_oid,
+              current_setting('block_size')::numeric AS bs,
+              /* MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) */
+              CASE
+                WHEN version() ~ 'mingw32' OR version() ~ '64-bit' THEN 8
+                ELSE 4
+              END AS maxalign,
+              24 AS pagehdr,
+              /* per tuple header: add index_attribute_bm if some cols are null-able */
+              CASE WHEN max(coalesce(s.null_frac,0)) = 0
+                THEN 2
+                ELSE 6
+              END AS index_tuple_hdr,
+              /* data len: we remove null values save space using it fractionnal part from stats */
+              sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 2048) ) AS nulldatawidth
+            FROM
+              pg_attribute AS a
+            JOIN
+              pg_stats AS s ON (quote_ident(s.schemaname) || '.' || quote_ident(s.tablename))::regclass=a.attrelid AND s.attname = a.attname
+            JOIN
+              btree_index_atts AS i ON i.indrelid = a.attrelid AND a.attnum = i.attnum
+            WHERE
+              a.attnum > 0
+            GROUP BY
+              1, 2, 3, 4, 5, 6, 7, 8, 9
+          ),
+          index_aligned AS (
+            SELECT
+              maxalign,
+              bs,
+              nspname,
+              relname AS index_name,
+              reltuples,
+              relpages,
+              relam,
+              table_oid,
+              index_oid,
+              ( 2 +
+                maxalign - CASE /* Add padding to the index tuple header to align on MAXALIGN */
+                  WHEN index_tuple_hdr%maxalign = 0 THEN maxalign
+                  ELSE index_tuple_hdr%maxalign
+                END
+              + nulldatawidth + maxalign - CASE /* Add padding to the data to align on MAXALIGN */
+                  WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
+                  ELSE nulldatawidth::integer%maxalign
+                END
+              )::numeric AS nulldatahdrwidth, pagehdr
+            FROM
+              index_item_sizes AS s1
+          ),
+          otta_calc AS (
+            SELECT
+              bs,
+              nspname,
+              table_oid,
+              index_oid,
+              index_name,
+              relpages,
+              coalesce(
+                ceil((reltuples*(4+nulldatahdrwidth))/(bs-pagehdr::float)) +
+                CASE WHEN am.amname IN ('hash','btree') THEN 1 ELSE 0 END , 0 /* btree and hash have a metadata reserved block */
+              ) AS otta
+            FROM
+              index_aligned AS s2
+            LEFT JOIN
+              pg_am am ON s2.relam = am.oid
+          ),
+          raw_bloat AS (
+            SELECT
+              nspname,
+              c.relname AS table_name,
+              index_name,
+              bs*(sub.relpages)::bigint AS totalbytes,
+              CASE
+                WHEN sub.relpages <= otta THEN 0
+                ELSE bs*(sub.relpages-otta)::bigint END
+                AS wastedbytes,
+              CASE
+                WHEN sub.relpages <= otta
+                THEN 0 ELSE bs*(sub.relpages-otta)::bigint * 100 / (bs*(sub.relpages)::bigint) END
+                AS realbloat,
+              pg_relation_size(sub.table_oid) as table_bytes,
+              stat.idx_scan as index_scans,
+              stat.indexrelid
+            FROM
+              otta_calc AS sub
+            JOIN
+              pg_class AS c ON c.oid=sub.table_oid
+            JOIN
+              pg_stat_user_indexes AS stat ON sub.index_oid = stat.indexrelid
+          )
+          SELECT
+            nspname AS schema,
+            table_name AS table,
+            index_name AS index,
+            wastedbytes AS bloat_bytes,
+            totalbytes AS index_bytes,
+            pg_get_indexdef(indexrelid) AS definition
+          FROM
+            raw_bloat
+          WHERE
+            wastedbytes >= #{min_size.to_i}
+          ORDER BY
+            wastedbytes DESC,
+            index_name
+        SQL
       end
     end
   end

data/lib/pghero/methods/kill.rb

@@ -2,11 +2,11 @@ module PgHero
   module Methods
     module Kill
       def kill(pid)
-        PgHero.truthy? execute("SELECT pg_terminate_backend(#{pid.to_i})").first["pg_terminate_backend"]
+        select_one("SELECT pg_terminate_backend(#{pid.to_i})")
       end
 
       def kill_long_running_queries
-        long_running_queries.each { |query| kill(query["pid"]) }
+        long_running_queries.each { |query| kill(query[:pid]) }
         true
       end
 

data/lib/pghero/methods/maintenance.rb

@@ -5,40 +5,32 @@ module PgHero
       # "the system will shut down and refuse to start any new transactions
       # once there are fewer than 1 million transactions left until wraparound"
       # warn when 10,000,000 transactions left
-      def transaction_id_danger(options = {})
-        threshold = options[:threshold] || 10000000
+      def transaction_id_danger(threshold: 10000000, max_value: 2146483648)
+        max_value = max_value.to_i
+        threshold = threshold.to_i
+
         select_all <<-SQL
           SELECT
-            c.oid::regclass::text AS table,
-            2146483648 - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) AS transactions_before_shutdown
+            n.nspname AS schema,
+            c.relname AS table,
+            #{quote(max_value)} - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) AS transactions_left
           FROM
             pg_class c
+          INNER JOIN
+            pg_catalog.pg_namespace n ON n.oid = c.relnamespace
           LEFT JOIN
             pg_class t ON c.reltoastrelid = t.oid
           WHERE
             c.relkind = 'r'
-            AND (2146483648 - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid))) < #{threshold}
+            AND (#{quote(max_value)} - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid))) < #{quote(threshold)}
           ORDER BY
            2, 1
         SQL
       end
 
       def autovacuum_danger
-        select_all <<-SQL
-          SELECT
-            c.oid::regclass::text as table,
-            (SELECT setting FROM pg_settings WHERE name = 'autovacuum_freeze_max_age')::int -
-            GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) AS transactions_before_autovacuum
-          FROM
-            pg_class c
-          LEFT JOIN
-            pg_class t ON c.reltoastrelid = t.oid
-          WHERE
-            c.relkind = 'r'
-            AND (SELECT setting FROM pg_settings WHERE name = 'autovacuum_freeze_max_age')::int - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) < 2000000
-          ORDER BY
-            transactions_before_autovacuum
-        SQL
+        max_value = select_one("SHOW autovacuum_freeze_max_age").to_i
+        transaction_id_danger(threshold: 2000000, max_value: max_value)
       end
 
       def maintenance_info
@@ -56,6 +48,28 @@ module PgHero
             1, 2
         SQL
       end
+
+      def analyze(table, verbose: false)
+        execute "ANALYZE #{verbose ? "VERBOSE " : ""}#{quote_table_name(table)}"
+        true
+      end
+
+      def analyze_tables(verbose: false, min_size: nil, tables: nil)
+        tables = table_stats(table: tables).reject { |s| %w(information_schema pg_catalog).include?(s[:schema]) }
+        tables = tables.select { |s| s[:size_bytes] > min_size } if min_size
+        tables.map { |s| s.slice(:schema, :table) }.each do |stats|
+          begin
+            with_transaction(lock_timeout: 5000, statement_timeout: 120000) do
+              analyze "#{stats[:schema]}.#{stats[:table]}", verbose: verbose
+            end
+            success = true
+          rescue ActiveRecord::StatementInvalid => e
+            $stderr.puts e.message
+            success = false
+          end
+          stats[:success] = success
+        end
+      end
     end
   end
 end

data/lib/pghero/methods/queries.rb

@@ -1,8 +1,7 @@
 module PgHero
   module Methods
     module Queries
-      def running_queries(options = {})
-        min_duration = options[:min_duration]
+      def running_queries(min_duration: nil)
         select_all <<-SQL
           SELECT
             pid,
@@ -12,6 +11,7 @@ module PgHero
             #{server_version_num >= 90600 ? "(wait_event IS NOT NULL) AS waiting" : "waiting"},
             query,
             COALESCE(query_start, xact_start) AS started_at,
+            EXTRACT(EPOCH FROM NOW() - COALESCE(query_start, xact_start)) * 1000.0 AS duration_ms,
             usename AS user
           FROM
             pg_stat_activity
@@ -30,55 +30,38 @@ module PgHero
         running_queries(min_duration: long_running_query_sec)
       end
 
-      def slow_queries(options = {})
-        query_stats = options[:query_stats] || self.query_stats(options.except(:query_stats))
-        query_stats.select { |q| q["calls"].to_i >= slow_query_calls.to_i && q["average_time"].to_i >= slow_query_ms.to_i }
-      end
-
-      def locks
-        select_all <<-SQL
-          SELECT DISTINCT ON (pid)
-            pg_stat_activity.pid,
-            pg_stat_activity.query,
-            age(now(), pg_stat_activity.query_start) AS age
-          FROM
-            pg_stat_activity
-          INNER JOIN
-            pg_locks ON pg_locks.pid = pg_stat_activity.pid
-          WHERE
-            pg_stat_activity.query <> '<insufficient privilege>'
-            AND pg_locks.mode = 'ExclusiveLock'
-            AND pg_stat_activity.pid <> pg_backend_pid()
-            AND pg_stat_activity.datname = current_database()
-          ORDER BY
-            pid,
-            query_start
-        SQL
-      end
-
       # from https://wiki.postgresql.org/wiki/Lock_Monitoring
+      # and http://big-elephants.com/2013-09/exploring-query-locks-in-postgres/
       def blocked_queries
         select_all <<-SQL
           SELECT
-            bl.pid AS blocked_pid,
-            a.usename AS blocked_user,
-            ka.query AS current_or_recent_query_in_blocking_process,
-            ka.state AS state_of_blocking_process,
-            age(now(), ka.query_start) AS blocking_duration,
-            kl.pid AS blocking_pid,
-            ka.usename AS blocking_user,
-            a.query AS blocked_query,
-            age(now(), a.query_start) AS blocked_duration
+            COALESCE(blockingl.relation::regclass::text,blockingl.locktype) as locked_item,
+            blockeda.pid AS blocked_pid,
+            blockeda.usename AS blocked_user,
+            blockeda.query as blocked_query,
+            age(now(), blockeda.query_start) AS blocked_duration,
+            blockedl.mode as blocked_mode,
+            blockinga.pid AS blocking_pid,
+            blockinga.usename AS blocking_user,
+            blockinga.state AS state_of_blocking_process,
+            blockinga.query AS current_or_recent_query_in_blocking_process,
+            age(now(), blockinga.query_start) AS blocking_duration,
+            blockingl.mode as blocking_mode
           FROM
-            pg_catalog.pg_locks bl
-          JOIN
-            pg_catalog.pg_stat_activity a ON a.pid = bl.pid
-          JOIN
-            pg_catalog.pg_locks kl ON kl.transactionid = bl.transactionid AND kl.pid != bl.pid
-          JOIN
-            pg_catalog.pg_stat_activity ka ON ka.pid = kl.pid
+            pg_catalog.pg_locks blockedl
+          LEFT JOIN
+            pg_stat_activity blockeda ON blockedl.pid = blockeda.pid
+          LEFT JOIN
+            pg_catalog.pg_locks blockingl ON blockedl.pid != blockingl.pid AND (
+              blockingl.transactionid = blockedl.transactionid
+              OR (blockingl.relation = blockedl.relation AND blockingl.locktype = blockedl.locktype)
+            )
+          LEFT JOIN
+            pg_stat_activity blockinga ON blockingl.pid = blockinga.pid AND blockinga.datid = blockeda.datid
           WHERE
-            NOT bl.GRANTED
+            NOT blockedl.granted
+            AND blockeda.query <> '<insufficient privilege>'
+            AND blockeda.datname = current_database()
           ORDER BY
             blocked_duration DESC
         SQL