pghero 2.0.8 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 553643782ed72cb0f44eb8e1f44efc239ace1075
-  data.tar.gz: 7defcc2212776fa9a45a45c206f8d7b5461a88c0
+  metadata.gz: f45d1e2c61994223e2835c0d8fdbefea6a9c0b36
+  data.tar.gz: 74227dbc52f80e9b1cb150f601a8ec9705ac56b3
 SHA512:
-  metadata.gz: 826e25c291d209bb7abd7b5f07ba99576136c6fccdfd60c6ddb075df038779b219958ac31edc50fbd316179a1ef45fc6a98ac284b31d804d7bebae60c07e1128
-  data.tar.gz: '0885bb30490a0003edabc433638c6de367ae312c603ed35178da12e1ac8fa497ef4e6d46c19c9c8640b07bdf062a1f755fe297166d7c4d3395c6fca8d8eb94e9'
+  metadata.gz: f96125273c33eb6b632894c927118c2e308a62829df66128ca3a917dd578eafae7dd77308fd832142ddb54d61e748d753ee06da83f5393d3417d447223a208cd
+  data.tar.gz: 777fc575408694f43f3a8de1273341c5be02e3288a63424e5150a13c5c4403fbc547c84b5172cdb23dc9720dfc3b50b8acf3d574dad399a569c43035df435984

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 2.1.0
+
+- Fixed issue with sequences in different schema than table
+- No longer throw errors for unreadable sequences
+- Fixed replication lag for Amazon Aurora
+- Added `vacuum_progress` method
+
 ## 2.0.8
 
 - Added support for Postgres 10 replicas

data/app/controllers/pg_hero/home_controller.rb

@@ -11,11 +11,13 @@ module PgHero
       before_action :set_database
       before_action :set_query_stats_enabled
       before_action :set_show_details, only: [:index, :queries, :show_query]
+      before_action :ensure_query_stats, only: [:queries]
     else
       # no need to check API in earlier versions
       before_filter :set_database
       before_filter :set_query_stats_enabled
       before_filter :set_show_details, only: [:index, :queries, :show_query]
+      before_filter :ensure_query_stats, only: [:queries]
     end
 
     def index
@@ -24,7 +26,7 @@ module PgHero
 
       if @replica
         @replication_lag = @database.replication_lag
-        @good_replication_lag = @replication_lag < 5
+        @good_replication_lag = @replication_lag ? @replication_lag < 5 : true
       else
         @inactive_replication_slots = @database.replication_slots.select { |r| !r[:active] }
       end
@@ -36,15 +38,9 @@ module PgHero
 
       @transaction_id_danger = @database.transaction_id_danger(threshold: 1500000000)
 
-      begin
-        @sequence_danger = @database.sequence_danger(threshold: (params[:sequence_threshold] || 0.9).to_f)
-      rescue ActiveRecord::StatementInvalid => e
-        if (m = /permission denied for [^:]+/.match(e.message))
-          @sequence_danger_error = m[0]
-        else
-          raise
-        end
-      end
+      @readable_sequences, @unreadable_sequences = @database.sequences.partition { |s| s[:readable] }
+
+      @sequence_danger = @database.sequence_danger(threshold: (params[:sequence_threshold] || 0.9).to_f, sequences: @readable_sequences)
 
       @indexes = @database.indexes
       @invalid_indexes = @indexes.select { |i| !i[:valid] }
@@ -112,6 +108,7 @@ module PgHero
     def live_queries
       @title = "Live Queries"
       @running_queries = @database.running_queries(all: true)
+      @vacuum_progress = @database.vacuum_progress.index_by { |q| q[:pid] }
     end
 
     def queries
@@ -385,5 +382,11 @@ module PgHero
         render text: message
       end
     end
+
+    def ensure_query_stats
+      unless @query_stats_enabled
+        redirect_to root_path, alert: "Query stats not enabled"
+      end
+    end
   end
 end

data/app/views/pg_hero/home/_live_queries_table.html.erb

@@ -12,7 +12,13 @@
       <tr>
         <td><%= query[:pid] %></td>
         <td><%= number_with_delimiter(query[:duration_ms].round) %> ms</td>
-        <td><%= query[:state] %></td>
+        <td>
+          <%= query[:state] %>
+          <% if vacuum_progress[query[:pid]] %>
+            <br />
+            <strong><%= vacuum_progress[query[:pid]][:phase] %></strong>
+          <% end %>
+        </td>
         <td class="text-right">
           <% button_path, button_options = Rails.version >= "4.1" ? [explain_path, {params: {query: query[:query]}}] : [explain_path(query: query[:query]), {}] %>
           <%= button_to "Explain", button_path, button_options.merge(form: {target: "_blank"}, class: "btn btn-info") %>

data/app/views/pg_hero/home/index.html.erb

@@ -1,12 +1,16 @@
 <div id="status">
   <% if @replica %>
     <div class="alert alert-<%= @good_replication_lag ? "success" : "warning" %>">
-      <% if @good_replication_lag %>
-        Healthy replication lag
+      <% if @replication_lag %>
+        <% if @good_replication_lag %>
+          Healthy replication lag
+        <% else %>
+          High replication lag
+        <% end %>
+        <span class="tiny"><%= number_with_delimiter((@replication_lag * 1000).round) %> ms</span>
       <% else %>
-        High replication lag
+        Replication lag not supported
       <% end %>
-      <span class="tiny"><%= number_with_delimiter((@replication_lag * 1000).round) %> ms</span>
     </div>
   <% elsif @inactive_replication_slots.any? %>
     <div class="alert alert-warning">
@@ -48,13 +52,14 @@
     <% end %>
   </div>
   <div class="alert alert-<%= @sequence_danger && @sequence_danger.empty? ? "success" : "warning" %>">
-    <% if !@sequence_danger %>
-      Error checking for columns near integer overflow: <%= @sequence_danger_error %>
-    <% elsif @sequence_danger.any? %>
-      <%= pluralize(@sequence_danger.size, "columns") %> approaching overflow
+    <% if @sequence_danger.any? %>
+      <%= pluralize(@sequence_danger.size, "column") %> approaching overflow
     <% else %>
       No columns near integer overflow
     <% end %>
+    <% if @unreadable_sequences.any? %>
+      (<%= link_to pluralize(@unreadable_sequences.size, "unreadable sequence", "unreadable sequences"), {unreadable: "t"} %>)
+    <% end %>
   </div>
   <div class="alert alert-<%= @invalid_indexes.empty? ? "success" : "warning" %>">
     <% if @invalid_indexes.any? %>
@@ -101,6 +106,41 @@
   <% end %>
 </div>
 
+<% if params[:unreadable] && @unreadable_sequences.any? %>
+  <div class="content">
+    <h1>Unreadable Sequences</h1>
+
+    <p>This is likely due to missing privileges. Make sure your user has the SELECT privilege for each sequence.</p>
+
+    <table class="table">
+      <thead>
+        <tr>
+          <th style="width: 33%;">Column</th>
+          <th>Sequence</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @unreadable_sequences.each do |sequence| %>
+          <tr>
+            <td>
+              <%= sequence[:table] %>.<%= sequence[:column] %>
+              <% if sequence[:table_schema] != "public" %>
+                <span class="text-muted"><%= sequence[:table_schema] %></span>
+              <% end %>
+            </td>
+            <td>
+              <%= sequence[:sequence] %>
+              <% if sequence[:schema] && sequence[:schema] != "public" %>
+                <span class="text-muted"><%= sequence[:schema] %></span>
+              <% end %>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% end %>
+
 <% if @replica && !@good_replication_lag %>
   <div class="content">
     <h1>High Replication Lag</h1>
@@ -141,7 +181,7 @@
 
     <pre><code>ALTER ROLE &lt;user&gt; SET statement_timeout TO '60s';</code></pre>
 
-    <%= render partial: "live_queries_table", locals: {queries: @long_running_queries} %>
+    <%= render partial: "live_queries_table", locals: {queries: @long_running_queries, vacuum_progress: {}} %>
   </div>
 <% end %>
 
@@ -224,6 +264,9 @@
           <tr>
             <td>
               <%= query[:table] %>.<%= query[:column] %>
+              <% if query[:table_schema] != "public" %>
+                <span class="text-muted"><%= query[:table_schema] %></span>
+              <% end %>
             </td>
             <td>
               <%= query[:column_type] %>

data/app/views/pg_hero/home/live_queries.html.erb

@@ -3,7 +3,7 @@
 
   <p><%= pluralize(@running_queries.size, "query") %></p>
 
-  <%= render partial: "live_queries_table", locals: {queries: @running_queries} %>
+  <%= render partial: "live_queries_table", locals: {queries: @running_queries, vacuum_progress: @vacuum_progress} %>
 
   <p><%= button_to "Kill all connections", kill_all_path, class: "btn btn-danger" %></p>
 

data/lib/pghero/methods/basic.rb

@@ -35,9 +35,21 @@ module PgHero
       def select_all(sql, conn = nil)
         conn ||= connection
         # squish for logs
-        result = conn.select_all(squish(sql))
-        cast_method = ActiveRecord::VERSION::MAJOR < 5 ? :type_cast : :cast_value
-        result.map { |row| Hash[row.map { |col, val| [col.to_sym, result.column_types[col].send(cast_method, val)] }] }
+        retries = 0
+        begin
+          result = conn.select_all(squish(sql))
+          cast_method = ActiveRecord::VERSION::MAJOR < 5 ? :type_cast : :cast_value
+          result.map { |row| Hash[row.map { |col, val| [col.to_sym, result.column_types[col].send(cast_method, val)] }] }
+        rescue ActiveRecord::StatementInvalid => e
+          # fix for random internal errors
+          if e.message.include?("PG::InternalError") && retries < 2
+            retries += 1
+            sleep(0.1)
+            retry
+          else
+            raise e
+          end
+        end
       end
 
       def select_all_stats(sql)

data/lib/pghero/methods/maintenance.rb

@@ -33,6 +33,22 @@ module PgHero
         transaction_id_danger(threshold: 2000000, max_value: max_value)
       end
 
+      def vacuum_progress
+        if server_version_num >= 90600
+          select_all <<-SQL
+            SELECT
+              pid,
+              phase
+            FROM
+              pg_stat_progress_vacuum
+            WHERE
+              datname = current_database()
+          SQL
+        else
+          []
+        end
+      end
+
       def maintenance_info
         select_all <<-SQL
           SELECT

data/lib/pghero/methods/replication.rb

@@ -9,22 +9,24 @@ module PgHero
       end
 
       # http://www.postgresql.org/message-id/CADKbJJWz9M0swPT3oqe8f9+tfD4-F54uE6Xtkh4nERpVsQnjnw@mail.gmail.com
-      def replication_lag        
-        lag_condition =
-          if server_version_num >= 100000
-            "pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()"
-          else
-            "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
-          end
+      def replication_lag
+        with_feature_support do
+          lag_condition =
+            if server_version_num >= 100000
+              "pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()"
+            else
+              "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
+            end
 
-        select_one <<-SQL
-          SELECT
-            CASE
-              WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
-              ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
-            END
-          AS replication_lag
-        SQL
+          select_one <<-SQL
+            SELECT
+              CASE
+                WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
+                ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
+              END
+            AS replication_lag
+          SQL
+        end
       end
 
       def replication_slots
@@ -51,7 +53,7 @@ module PgHero
 
       private
 
-      def with_feature_support(default)
+      def with_feature_support(default = nil)
         begin
           yield
         rescue ActiveRecord::StatementInvalid => e

data/lib/pghero/methods/sequences.rb

@@ -29,31 +29,79 @@ module PgHero
         # parse out sequence
         sequences.each do |column|
           m = /^nextval\('(.+)'\:\:regclass\)$/.match(column.delete(:default_value))
-          column[:schema], column[:sequence] = unquote_ident(m[1], column[:table_schema])
+          column[:schema], column[:sequence] = unquote_ident(m[1])
           column[:max_value] = column[:column_type] == 'integer' ? 2147483647 : 9223372036854775807
         end
 
-        select_all(sequences.map { |s| "SELECT last_value FROM #{quote_ident(s[:schema])}.#{quote_ident(s[:sequence])}" }.join(" UNION ALL ")).each_with_index do |row, i|
+        add_sequence_attributes(sequences)
+
+        select_all(sequences.select { |s| s[:readable] }.map { |s| "SELECT last_value FROM #{quote_ident(s[:schema])}.#{quote_ident(s[:sequence])}" }.join(" UNION ALL ")).each_with_index do |row, i|
           sequences[i][:last_value] = row[:last_value]
         end
 
         sequences.sort_by { |s| s[:sequence] }
       end
 
-      def sequence_danger(threshold: 0.9)
-        sequences.select { |s| s[:last_value] / s[:max_value].to_f > threshold }.sort_by { |s| s[:max_value] - s[:last_value] }
+      def sequence_danger(threshold: 0.9, sequences: nil)
+        sequences ||= self.sequences
+        sequences.select { |s| s[:last_value] && s[:last_value] / s[:max_value].to_f > threshold }.sort_by { |s| s[:max_value] - s[:last_value] }
       end
 
       private
 
-      def unquote_ident(value, default_schema)
+      def unquote_ident(value)
         schema, seq = value.split(".")
         unless seq
           seq = schema
-          schema = default_schema
+          schema = nil
         end
         [unquote(schema), unquote(seq)]
       end
+
+      # adds readable attribute to all sequences
+      # also adds schema if missing
+      def add_sequence_attributes(sequences)
+        # fetch data
+        sequence_attributes = select_all <<-SQL
+          SELECT
+            n.nspname AS schema,
+            c.relname AS sequence,
+            has_sequence_privilege(c.oid, 'SELECT') AS readable
+          FROM
+            pg_class c
+          INNER JOIN
+            pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+          WHERE
+            c.relkind = 'S'
+            AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+        SQL
+
+        # first populate missing schemas
+        missing_schema = sequences.select { |s| s[:schema].nil? }
+        if missing_schema.any?
+          sequence_schemas = sequence_attributes.group_by { |s| s[:sequence] }
+
+          missing_schema.each do |sequence|
+            schemas = sequence_schemas[sequence[:sequence]] || []
+
+            case schemas.size
+            when 0
+              # do nothing, will be marked as unreadable
+            when 1
+              # bingo
+              sequence[:schema] = schemas[0][:schema]
+            else
+              raise PgHero::Error, "Same sequence name in multiple schemas: #{sequence[:sequence]}"
+            end
+          end
+        end
+
+        # then populate attributes
+        readable = Hash[sequence_attributes.map { |s| [[s[:schema], s[:sequence]], s[:readable]] }]
+        sequences.each do |sequence|
+          sequence[:readable] = readable[[sequence[:schema], sequence[:sequence]]] || false
+        end
+      end
     end
   end
 end

data/lib/pghero/version.rb

@@ -1,3 +1,3 @@
 module PgHero
-  VERSION = "2.0.8"
+  VERSION = "2.1.0"
 end

data/lib/pghero.rb

@@ -109,6 +109,7 @@ module PgHero
 
     def capture_query_stats(verbose: false)
       each_database do |database|
+        next unless database.capture_query_stats?
         puts "Capturing query stats for #{database.id}..." if verbose
         database.capture_query_stats(raise_errors: true)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pghero
 version: !ruby/object:Gem::Version
-  version: 2.0.8
+  version: 2.1.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-13 00:00:00.000000000 Z
+date: 2017-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord