pghero 2.4.1 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1eb28c7ae995ccdc8fcc30dec3c42f3abb1463981b89a85ed9a8b8b53dba4ce
-  data.tar.gz: 0c0b1e34f2f12c46055a15c937b282048b99036ce6b535ac2445bd83b4d7ac67
+  metadata.gz: 38e408389854af6f5a2a71ee70bc3d66c257a299b1a1fec255c279819db00482
+  data.tar.gz: 2f492cf50fd99032ec9ac9cfd03df70ef5c4d45ea55a0c14a5c608102b5aa106
 SHA512:
-  metadata.gz: fc924cde46b89d3d64649f8df7abbe81538892ba0839c4983ad0e67522c36ee4b0ee52673c3d79977cfac05413c846d821fb25d885ba33693e2d01c6de042f8b
-  data.tar.gz: 790d0cc392a123ac30ae59868a80eb1239adbcf091a53c0edd9cbabb6426f0e86d11d25ac8fbc0c85619c90f766c46a67a49e439467b8c47f960482e8d04c10d
+  metadata.gz: edc72b96fb3c766f41682c22aa2287a99c610435687794d0a09395821a129709b0e464f7e9031d43cac7a6d1bc334cb419885dc32b87d9d4e58a75d810c3ba30
+  data.tar.gz: a4ad92136153cb3f607bddb0b289d3c180a6a653f7a6eac9b47ca6fe7fa04626923f7b84e80951577c9a94314c2487ac7dccb4cdd5f2483a3076473893bac13e

data/CHANGELOG.md

@@ -1,3 +1,33 @@
+## 2.7.0 (2020-08-04)
+
+- Fixed CSRF vulnerability with non-session based authentication
+- Added `database`, `user`, and `query_hash` options to `reset_query_stats` method
+
+## 2.6.0 (2020-07-09)
+
+- Added support for Postgres 13 beta 2
+- Added support for non-integer explain timeout
+
+## 2.5.1 (2020-06-23)
+
+- Added support for `google-cloud-monitoring` >= 1
+- Added support for `google-cloud-monitoring-v3`
+- Fixed system stats not showing up in Rails 6 with environment variables
+
+## 2.5.0 (2020-05-24)
+
+- Added system stats for Google Cloud SQL and Azure Database
+- Added experimental `filter_data` option
+- Localized times on maintenance page
+- Improved connection pooling
+- Improved error message for non-Postgres connections
+- Fixed more deprecation warnings in Ruby 2.7
+
+## 2.4.2 (2020-04-16)
+
+- Added `connections` method
+- Fixed deprecation warnings in Ruby 2.7
+
 ## 2.4.1 (2019-11-21)
 
 - Fixed file permissions on `highlight.pack.js`

data/README.md

@@ -10,17 +10,21 @@ A performance dashboard for Postgres
 
 ---
 
-[![Screenshot](https://pghero.dokkuapp.com/assets/screenshot-5a368624ada55b32e7668c96926840f9.png)](https://pghero.dokkuapp.com/)
+[![Screenshot](https://pghero.dokkuapp.com/assets/pghero-f8abe426e6bf54bb7dba87b425bb809740ebd386208bcd280a7e802b053a1023.png)](https://pghero.dokkuapp.com/)
+
+:tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
+
+[![Build Status](https://travis-ci.org/ankane/pghero.svg?branch=master)](https://travis-ci.org/ankane/pghero) [![Docker Pulls](https://img.shields.io/docker/pulls/ankane/pghero)](https://hub.docker.com/repository/docker/ankane/pghero)
 
 ## Installation
 
-PgHero is available as a Rails engine, Linux package, and Docker image.
+PgHero is available as a Docker image, Linux package, and Rails engine.
 
 Select your preferred method of installation to get started.
 
-- [Rails](guides/Rails.md)
-- [Linux](guides/Linux.md)
 - [Docker](guides/Docker.md)
+- [Linux](guides/Linux.md)
+- [Rails](guides/Rails.md)
 
 ## Related Projects
 
@@ -31,10 +35,17 @@ Select your preferred method of installation to get started.
 
 ## Credits
 
-A big thanks to [Craig Kerstiens](http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/) and [Heroku](https://blog.heroku.com/archives/2013/5/10/more_insight_into_your_database_with_pgextras) for the initial queries and [Bootswatch](https://github.com/thomaspark/bootswatch) for the theme :clap:
+A big thanks to [Craig Kerstiens](http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/) and [Heroku](https://blog.heroku.com/archives/2013/5/10/more_insight_into_your_database_with_pgextras) for the initial queries and [Bootswatch](https://github.com/thomaspark/bootswatch) for the theme.
 
-Know a bit about PostgreSQL? [Suggestions](https://github.com/ankane/pghero/issues) are greatly appreciated.
+## History
 
-:tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
+View the [changelog](https://github.com/ankane/pghero/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
 
-[![Build Status](https://travis-ci.org/ankane/pghero.svg?branch=master)](https://travis-ci.org/ankane/pghero)
+- [Report bugs](https://github.com/ankane/pghero/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/pghero/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features

data/app/assets/javascripts/pghero/application.js

@@ -59,7 +59,7 @@ function initSlider() {
         html = "Now";
       }
     } else {
-      html = months[time.getMonth()] + " " + time.getDate() + ", " + pad(time.getHours()) + ":" + pad(time.getMinutes());
+      html = time.getDate() + " " + months[time.getMonth()] + " " + pad(time.getHours()) + ":" + pad(time.getMinutes());
     }
     $(selector).html(html);
   }

data/app/controllers/pg_hero/home_controller.rb

@@ -2,7 +2,7 @@ module PgHero
   class HomeController < ActionController::Base
     layout "pg_hero/application"
 
-    protect_from_forgery
+    protect_from_forgery with: :exception
 
     http_basic_authenticate_with name: PgHero.username, password: PgHero.password if PgHero.password
 
@@ -13,6 +13,11 @@ module PgHero
     before_action :ensure_query_stats, only: [:queries]
 
     if PgHero.config["override_csp"]
+      # note: this does not take into account asset hosts
+      # which can be a string with %d or a proc
+      # https://api.rubyonrails.org/classes/ActionView/Helpers/AssetUrlHelper.html
+      # users should set CSP manually if needed
+      # see https://github.com/ankane/pghero/issues/297
       after_action do
         response.headers["Content-Security-Policy"] = "default-src 'self' 'unsafe-inline'"
       end
@@ -198,6 +203,11 @@ module PgHero
         "1 week" => {duration: 1.week, period: 30.minutes},
         "2 weeks" => {duration: 2.weeks, period: 1.hours}
       }
+      if @database.system_stats_provider == :azure
+        # doesn't support 10, just 5 and 15
+        @periods["1 day"][:period] = 15.minutes
+      end
+
       @duration = (params[:duration] || 1.hour).to_i
       @period = (params[:period] || 60.seconds).to_i
 
@@ -209,22 +219,36 @@ module PgHero
     end
 
     def cpu_usage
-      render json: [{name: "CPU", data: @database.cpu_usage(system_params).map { |k, v| [k, v.round] }, library: chart_library_options}]
+      render json: [{name: "CPU", data: @database.cpu_usage(**system_params).map { |k, v| [k, v ? v.round : v] }, library: chart_library_options}]
     end
 
     def connection_stats
-      render json: [{name: "Connections", data: @database.connection_stats(system_params), library: chart_library_options}]
+      render json: [{name: "Connections", data: @database.connection_stats(**system_params), library: chart_library_options}]
     end
 
     def replication_lag_stats
-      render json: [{name: "Lag", data: @database.replication_lag_stats(system_params), library: chart_library_options}]
+      render json: [{name: "Lag", data: @database.replication_lag_stats(**system_params), library: chart_library_options}]
     end
 
     def load_stats
-      render json: [
-        {name: "Read IOPS", data: @database.read_iops_stats(system_params).map { |k, v| [k, v.round] }, library: chart_library_options},
-        {name: "Write IOPS", data: @database.write_iops_stats(system_params).map { |k, v| [k, v.round] }, library: chart_library_options}
-      ]
+      stats =
+        case @database.system_stats_provider
+        when :azure
+          [
+            {name: "IO Consumption", data: @database.azure_stats("io_consumption_percent", **system_params), library: chart_library_options}
+          ]
+        when :gcp
+          [
+            {name: "Read Ops", data: @database.read_iops_stats(**system_params).map { |k, v| [k, v ? v.round : v] }, library: chart_library_options},
+            {name: "Write Ops", data: @database.write_iops_stats(**system_params).map { |k, v| [k, v ? v.round : v] }, library: chart_library_options}
+          ]
+        else
+          [
+            {name: "Read IOPS", data: @database.read_iops_stats(**system_params).map { |k, v| [k, v ? v.round : v] }, library: chart_library_options},
+            {name: "Write IOPS", data: @database.write_iops_stats(**system_params).map { |k, v| [k, v ? v.round : v] }, library: chart_library_options}
+          ]
+        end
+      render json: stats
     end
 
     def free_space_stats
@@ -270,17 +294,48 @@ module PgHero
 
     def connections
       @title = "Connections"
-      @connection_sources = @database.connection_sources
-      @total_connections = @connection_sources.sum { |cs| cs[:total_connections] }
+      connections = @database.connections
+
+      @total_connections = connections.count
+      @connection_sources = group_connections(connections, [:database, :user, :source, :ip])
+      @connections_by_database = group_connections_by_key(connections, :database)
+      @connections_by_user = group_connections_by_key(connections, :user)
+
+      if params[:security] && @database.server_version_num >= 90500
+        connections.each do |connection|
+          connection[:ssl_status] =
+            if connection[:ssl]
+              # no way to tell if client used verify-full
+              # so connection may not be actually secure
+              "SSL"
+            else
+              # variety of reasons for no SSL
+              if !connection[:database].present?
+                "Internal Process"
+              elsif !connection[:ip]
+                if connection[:state]
+                  "Socket"
+                else
+                  # tcp or socket, don't have permission to tell
+                  "No SSL"
+                end
+              else
+                # tcp
+                # could separate out localhost since this should be safe
+                "No SSL"
+              end
+            end
+        end
 
-      @connections_by_database = group_connections(@connection_sources, :database)
-      @connections_by_user = group_connections(@connection_sources, :user)
+        @connections_by_ssl_status = group_connections_by_key(connections, :ssl_status)
+      end
     end
 
     def maintenance
       @title = "Maintenance"
       @maintenance_info = @database.maintenance_info
       @time_zone = PgHero.time_zone
+      @show_dead_rows = params[:dead_rows]
     end
 
     def kill
@@ -363,7 +418,8 @@ module PgHero
     def system_params
       {
         duration: params[:duration],
-        period: params[:period]
+        period: params[:period],
+        series: true
       }.delete_if { |_, v| v.nil? }
     end
 
@@ -376,18 +432,22 @@ module PgHero
       @show_details = @historical_query_stats_enabled && @database.supports_query_hash?
     end
 
-    def group_connections(connection_sources, key)
-      top_connections = Hash.new(0)
-      connection_sources.each do |source|
-        top_connections[source[key]] += source[:total_connections]
-      end
-      top_connections.sort_by { |k, v| [-v, k] }
+    def group_connections(connections, keys)
+      connections
+        .group_by { |conn| conn.slice(*keys) }
+        .map { |k, v| k.merge(total_connections: v.count) }
+        .sort_by { |v| [-v[:total_connections]] + keys.map { |k| v[k].to_s } }
+    end
+
+    def group_connections_by_key(connections, key)
+      group_connections(connections, [key]).map { |v| [v[key], v[:total_connections]] }.to_h
     end
 
     def check_api
       render_text "No support for Rails API. See https://github.com/pghero/pghero for a standalone app." if Rails.application.config.try(:api_only)
     end
 
+    # TODO return error status code
     def render_text(message)
       render plain: message
     end

data/app/helpers/pg_hero/home_helper.rb

@@ -15,5 +15,16 @@ module PgHero
     def pghero_js_var(name, value)
       "var #{name} = #{json_escape(value.to_json(root: false))};".html_safe
     end
+
+    def pghero_remove_index(query)
+      if query[:columns]
+        columns = query[:columns].map(&:to_sym)
+        columns = columns.first if columns.size == 1
+      end
+      ret = String.new("remove_index #{query[:table].to_sym.inspect}")
+      ret << ", name: #{(query[:name] || query[:index]).to_s.inspect}"
+      ret << ", column: #{columns.inspect}" if columns
+      ret
+    end
   end
 end

data/app/views/pg_hero/home/_live_queries_table.html.erb

@@ -30,7 +30,9 @@
           <% end %>
         </td>
         <td class="text-right">
-          <%= button_to "Explain", explain_path, params: {query: query[:query]}, form: {target: "_blank"}, class: "btn btn-info" %>
+          <% unless @database.filter_data %>
+            <%= button_to "Explain", explain_path, params: {query: query[:query]}, form: {target: "_blank"}, class: "btn btn-info" %>
+          <% end %>
           <%= button_to "Kill", kill_path(pid: query[:pid]), class: "btn btn-danger" %>
         </td>
       </tr>

data/app/views/pg_hero/home/connections.html.erb

@@ -18,6 +18,15 @@
       new Chartkick.PieChart("chart-2", <%= json_escape(@connections_by_user.to_json).html_safe %>);
     </script>
 
+    <% if @connections_by_ssl_status %>
+      <h3>By Security</h3>
+
+      <div id="chart-3" class="chart" style="height: 260px; line-height: 260px; margin-bottom: 20px;">Loading...</div>
+      <script>
+        new Chartkick.PieChart("chart-3", <%= json_escape(@connections_by_ssl_status.to_json).html_safe %>);
+      </script>
+    <% end %>
+
     <%= render partial: "connections_table", locals: {connection_sources: @connection_sources} %>
   <% end %>
 </div>

data/app/views/pg_hero/home/index.html.erb

@@ -387,7 +387,7 @@
       <pre>rails generate migration remove_unneeded_indexes</pre>
       <p>And paste</p>
       <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @duplicate_indexes.each do |query| %>
-remove_index <%= query[:unneeded_index][:table].to_sym.inspect %>, name: <%= query[:unneeded_index][:name].to_s.inspect %><% end %></pre>
+<%= pghero_remove_index(query[:unneeded_index]) %><% end %></pre>
     </div>
 
     <table class="table duplicate-indexes">
@@ -491,7 +491,7 @@ pg_stat_statements.track = all</pre>
       <pre>rails generate migration remove_unused_indexes</pre>
       <p>And paste</p>
       <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @unused_indexes.each do |query| %>
-remove_index <%= query[:table].to_sym.inspect %>, name: <%= query[:index].to_s.inspect %><% end %></pre>
+<%= pghero_remove_index(query)%><% end %></pre>
     </div>
 
     <table class="table">

data/app/views/pg_hero/home/maintenance.html.erb

@@ -7,6 +7,9 @@
         <th>Table</th>
         <th style="width: 20%;">Last Vacuum</th>
         <th style="width: 20%;">Last Analyze</th>
+        <% if @show_dead_rows %>
+          <th style="width: 20%;">Dead Rows</th>
+        <% end %>
       </tr>
     </thead>
     <tbody>
@@ -21,7 +24,7 @@
           <td>
             <% time = [table[:last_autovacuum], table[:last_vacuum]].compact.max %>
             <% if time %>
-              <%= time.in_time_zone(@time_zone).strftime("%-m/%-e %l:%M %P") %>
+              <%= l time.in_time_zone(@time_zone), format: :short %>
             <% else %>
               <span class="text-muted">Unknown</span>
             <% end %>
@@ -29,11 +32,22 @@
           <td>
             <% time = [table[:last_autoanalyze], table[:last_analyze]].compact.max %>
             <% if time %>
-              <%= time.in_time_zone(@time_zone).strftime("%-m/%-e %l:%M %P") %>
+              <%= l time.in_time_zone(@time_zone), format: :short %>
             <% else %>
               <span class="text-muted">Unknown</span>
             <% end %>
           </td>
+          <% if @show_dead_rows %>
+            <td>
+              <% if table[:live_rows] != 0 %>
+                <%# use live rows only for denominator to make it easier to compare with autovacuum_vacuum_scale_factor %>
+                <%# it's not a true percentage, since it can go above 100% %>
+                <%= (100.0 * table[:dead_rows] / table[:live_rows]).round %>%
+              <% else %>
+                <span class="text-muted">Unknown</span>
+              <% end %>
+            </td>
+          <% end %>
         </tr>
       <% end %>
     </tbody>

data/app/views/pg_hero/home/space.html.erb

@@ -33,7 +33,7 @@
       <pre>rails generate migration remove_unused_indexes</pre>
       <p>And paste</p>
       <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @unused_indexes.sort_by { |q| [-q[:size_bytes], q[:index]] }.each do |query| %>
-remove_index <%= query[:table].to_sym.inspect %>, name: <%= query[:index].to_s.inspect %><% end %></pre>
+<%= pghero_remove_index(query) %><% end %></pre>
     </div>
   <% end %>
 

data/app/views/pg_hero/home/tune.html.erb

@@ -18,7 +18,8 @@
     </tbody>
   </table>
 
-  <p>Check out <%= link_to "PgTune", "https://pgtune.leopard.in.ua/", target: "_blank" %> for recommendations. DB version is <%= @database.server_version.split(" ").first.split(".").first(2).join(".") %>.</p>
+  <% version_parts = @database.server_version.split(" ").first.split(".") %>
+  <p>Check out <%= link_to "PgTune", "https://pgtune.leopard.in.ua/", target: "_blank" %> for recommendations. DB version is <%= version_parts[0].to_i >= 10 ? version_parts[0] : version_parts.first(2).join(".") %>.</p>
 </div>
 
 <% if @autovacuum_settings %>

data/lib/generators/pghero/templates/config.yml.tt

@@ -3,6 +3,11 @@ databases:
     # Database URL (defaults to app database)
     # url: <%%= ENV["DATABASE_URL"] %>
 
+    # System stats
+    # aws_db_instance_identifier: my-instance
+    # gcp_database_id: my-project:my-instance
+    # azure_resource_id: my-resource-id
+
   # Add more databases
   # other:
   #   url: <%%= ENV["OTHER_DATABASE_URL"] %>
@@ -27,13 +32,15 @@ databases:
 
 # Basic authentication
 # username: admin
-# password: secret
+# password: <%%= ENV["PGHERO_PASSWORD"] %>
 
 # Stats database URL (defaults to app database)
 # stats_database_url: <%%= ENV["PGHERO_STATS_DATABASE_URL"] %>
 
 # AWS configuration (defaults to app AWS config)
-# also need aws_db_instance_identifier with each database
-# aws_access_key_id: ...
-# aws_secret_access_key: ...
+# aws_access_key_id: <%%= ENV["AWS_ACCESS_KEY_ID"] %>
+# aws_secret_access_key: <%%= ENV["AWS_SECRET_ACCESS_KEY"] %>
 # aws_region: us-east-1
+
+# Filter data from queries (experimental)
+# filter_data: true

data/lib/pghero.rb

@@ -34,24 +34,27 @@ module PgHero
   class Error < StandardError; end
   class NotEnabled < Error; end
 
+  MUTEX = Mutex.new
+
   # settings
   class << self
-    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :explain_timeout_sec, :total_connections_threshold, :cache_hit_rate_threshold, :env, :show_migrations, :config_path
+    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :explain_timeout_sec, :total_connections_threshold, :cache_hit_rate_threshold, :env, :show_migrations, :config_path, :filter_data
   end
   self.long_running_query_sec = (ENV["PGHERO_LONG_RUNNING_QUERY_SEC"] || 60).to_i
   self.slow_query_ms = (ENV["PGHERO_SLOW_QUERY_MS"] || 20).to_i
   self.slow_query_calls = (ENV["PGHERO_SLOW_QUERY_CALLS"] || 100).to_i
-  self.explain_timeout_sec = (ENV["PGHERO_EXPLAIN_TIMEOUT_SEC"] || 10).to_i
+  self.explain_timeout_sec = (ENV["PGHERO_EXPLAIN_TIMEOUT_SEC"] || 10).to_f
   self.total_connections_threshold = (ENV["PGHERO_TOTAL_CONNECTIONS_THRESHOLD"] || 500).to_i
   self.cache_hit_rate_threshold = 99
   self.env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
   self.show_migrations = true
   self.config_path = ENV["PGHERO_CONFIG_PATH"] || "config/pghero.yml"
+  self.filter_data = ENV["PGHERO_FILTER_DATA"].to_s.size > 0
 
   class << self
     extend Forwardable
     def_delegators :primary_database, :access_key_id, :analyze, :analyze_tables, :autoindex, :autovacuum_danger,
-      :best_index, :blocked_queries, :connection_sources, :connection_states, :connection_stats,
+      :best_index, :blocked_queries, :connections, :connection_sources, :connection_states, :connection_stats,
       :cpu_usage, :create_user, :database_size, :db_instance_identifier, :disable_query_stats, :drop_user,
       :duplicate_indexes, :enable_query_stats, :explain, :historical_query_stats_enabled?, :index_caching,
       :index_hit_rate, :index_usage, :indexes, :invalid_constraints, :invalid_indexes, :kill, :kill_all, :kill_long_running_queries,
@@ -116,11 +119,18 @@ module PgHero
 
           if databases.empty?
             databases["primary"] = {
-              "url" => ENV["PGHERO_DATABASE_URL"] || ActiveRecord::Base.connection_config,
-              "db_instance_identifier" => ENV["PGHERO_DB_INSTANCE_IDENTIFIER"]
+              "url" => ENV["PGHERO_DATABASE_URL"] || ActiveRecord::Base.connection_config
             }
           end
 
+          if databases.size == 1
+            databases.values.first.merge!(
+              "db_instance_identifier" => ENV["PGHERO_DB_INSTANCE_IDENTIFIER"],
+              "gcp_database_id" => ENV["PGHERO_GCP_DATABASE_ID"],
+              "azure_resource_id" => ENV["PGHERO_AZURE_RESOURCE_ID"]
+            )
+          end
+
           {
             "databases" => databases
           }
@@ -128,14 +138,20 @@ module PgHero
       end
     end
 
+    # ensure we only have one copy of databases
+    # so there's only one connection pool per database
     def databases
-      @databases ||= begin
-        Hash[
-          config["databases"].map do |id, c|
-            [id.to_sym, PgHero::Database.new(id, c)]
-          end
-        ]
+      unless defined?(@databases)
+        # only use mutex on initialization
+        MUTEX.synchronize do
+          # return if another process initialized while we were waiting
+          return @databases if defined?(@databases)
+
+          @databases = config["databases"].map { |id, c| [id.to_sym, Database.new(id, c)] }.to_h
+        end
       end
+
+      @databases
     end
 
     def primary_database
@@ -180,13 +196,13 @@ module PgHero
     # stats for old databases are not cleaned up since we can't use an index
     def clean_query_stats
       each_database do |database|
-        PgHero::QueryStats.where(database: database.id).where("captured_at < ?", 14.days.ago).delete_all
+        database.clean_query_stats
       end
     end
 
     def clean_space_stats
       each_database do |database|
-        PgHero::SpaceStats.where(database: database.id).where("captured_at < ?", 90.days.ago).delete_all
+        database.clean_space_stats
       end
     end