pghero 2.5.0 → 2.7.2

data/app/assets/stylesheets/pghero/application.css

@@ -508,3 +508,7 @@ body {
 .duplicate-indexes pre {
   margin-top: 10px;
 }
+
+.no-outline:focus {
+  outline: none;
+}

data/app/assets/stylesheets/pghero/nouislider.css

@@ -1,4 +1,4 @@
-/*! nouislider - 14.0.3 - 10/10/2019 */
+/*! nouislider - 14.6.1 - 8/17/2020 */
 /* Functional styling;
  * These styles are required for noUiSlider to function.
  * You don't need to change these rules to apply your design.
@@ -18,7 +18,6 @@
 }
 .noUi-target {
   position: relative;
-  direction: ltr;
 }
 .noUi-base,
 .noUi-connects {
@@ -39,7 +38,7 @@
   position: absolute;
   z-index: 1;
   top: 0;
-  left: 0;
+  right: 0;
   -ms-transform-origin: 0 0;
   -webkit-transform-origin: 0 0;
   -webkit-transform-style: preserve-3d;
@@ -56,9 +55,9 @@
 }
 /* Offset direction
  */
-html:not([dir="rtl"]) .noUi-horizontal .noUi-origin {
-  left: auto;
-  right: 0;
+.noUi-txt-dir-rtl.noUi-horizontal .noUi-origin {
+  left: 0;
+  right: auto;
 }
 /* Give origins 0 height/width so they don't interfere with clicking the
  * connect elements.
@@ -94,7 +93,7 @@ html:not([dir="rtl"]) .noUi-horizontal .noUi-origin {
 .noUi-horizontal .noUi-handle {
   width: 34px;
   height: 28px;
-  left: -17px;
+  right: -17px;
   top: -6px;
 }
 .noUi-vertical {
@@ -103,12 +102,12 @@ html:not([dir="rtl"]) .noUi-horizontal .noUi-origin {
 .noUi-vertical .noUi-handle {
   width: 28px;
   height: 34px;
-  left: -6px;
+  right: -6px;
   top: -17px;
 }
-html:not([dir="rtl"]) .noUi-horizontal .noUi-handle {
-  right: -17px;
-  left: auto;
+.noUi-txt-dir-rtl.noUi-horizontal .noUi-handle {
+  left: -17px;
+  right: auto;
 }
 /* Styling;
  * Giving the connect element a border radius causes issues with using transform: scale
@@ -297,3 +296,15 @@ html:not([dir="rtl"]) .noUi-horizontal .noUi-handle {
   top: 50%;
   right: 120%;
 }
+.noUi-horizontal .noUi-origin > .noUi-tooltip {
+  -webkit-transform: translate(50%, 0);
+  transform: translate(50%, 0);
+  left: auto;
+  bottom: 10px;
+}
+.noUi-vertical .noUi-origin > .noUi-tooltip {
+  -webkit-transform: translate(0, -18px);
+  transform: translate(0, -18px);
+  top: auto;
+  right: 28px;
+}

data/app/controllers/pg_hero/home_controller.rb

@@ -2,7 +2,7 @@ module PgHero
   class HomeController < ActionController::Base
     layout "pg_hero/application"
 
-    protect_from_forgery
+    protect_from_forgery with: :exception
 
     http_basic_authenticate_with name: PgHero.username, password: PgHero.password if PgHero.password
 

data/lib/generators/pghero/templates/config.yml.tt

@@ -32,14 +32,14 @@ databases:
 
 # Basic authentication
 # username: admin
-# password: secret
+# password: <%%= ENV["PGHERO_PASSWORD"] %>
 
 # Stats database URL (defaults to app database)
 # stats_database_url: <%%= ENV["PGHERO_STATS_DATABASE_URL"] %>
 
 # AWS configuration (defaults to app AWS config)
-# aws_access_key_id: ...
-# aws_secret_access_key: ...
+# aws_access_key_id: <%%= ENV["AWS_ACCESS_KEY_ID"] %>
+# aws_secret_access_key: <%%= ENV["AWS_SECRET_ACCESS_KEY"] %>
 # aws_region: us-east-1
 
 # Filter data from queries (experimental)

data/lib/pghero.rb

@@ -43,7 +43,7 @@ module PgHero
   self.long_running_query_sec = (ENV["PGHERO_LONG_RUNNING_QUERY_SEC"] || 60).to_i
   self.slow_query_ms = (ENV["PGHERO_SLOW_QUERY_MS"] || 20).to_i
   self.slow_query_calls = (ENV["PGHERO_SLOW_QUERY_CALLS"] || 100).to_i
-  self.explain_timeout_sec = (ENV["PGHERO_EXPLAIN_TIMEOUT_SEC"] || 10).to_i
+  self.explain_timeout_sec = (ENV["PGHERO_EXPLAIN_TIMEOUT_SEC"] || 10).to_f
   self.total_connections_threshold = (ENV["PGHERO_TOTAL_CONNECTIONS_THRESHOLD"] || 500).to_i
   self.cache_hit_rate_threshold = 99
   self.env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
@@ -119,11 +119,16 @@ module PgHero
 
           if databases.empty?
             databases["primary"] = {
-              "url" => ENV["PGHERO_DATABASE_URL"] || ActiveRecord::Base.connection_config,
+              "url" => ENV["PGHERO_DATABASE_URL"] || ActiveRecord::Base.connection_config
+            }
+          end
+
+          if databases.size == 1
+            databases.values.first.merge!(
               "db_instance_identifier" => ENV["PGHERO_DB_INSTANCE_IDENTIFIER"],
               "gcp_database_id" => ENV["PGHERO_GCP_DATABASE_ID"],
               "azure_resource_id" => ENV["PGHERO_AZURE_RESOURCE_ID"]
-            }
+            )
           end
 
           {
@@ -191,13 +196,13 @@ module PgHero
     # stats for old databases are not cleaned up since we can't use an index
     def clean_query_stats
       each_database do |database|
-        PgHero::QueryStats.where(database: database.id).where("captured_at < ?", 14.days.ago).delete_all
+        database.clean_query_stats
       end
     end
 
     def clean_space_stats
       each_database do |database|
-        PgHero::SpaceStats.where(database: database.id).where("captured_at < ?", 90.days.ago).delete_all
+        database.clean_space_stats
       end
     end
 

data/lib/pghero/database.rb

@@ -55,15 +55,16 @@ module PgHero
     end
 
     def explain_timeout_sec
-      (config["explain_timeout_sec"] || PgHero.config["explain_timeout_sec"] || PgHero.explain_timeout_sec).to_i
+      (config["explain_timeout_sec"] || PgHero.config["explain_timeout_sec"] || PgHero.explain_timeout_sec).to_f
     end
 
     def long_running_query_sec
       (config["long_running_query_sec"] || PgHero.config["long_running_query_sec"] || PgHero.long_running_query_sec).to_i
     end
 
+    # defaults to 100 megabytes
     def index_bloat_bytes
-      (config["index_bloat_bytes"] || PgHero.config["index_bloat_bytes"] || 100.megabytes).to_i
+      (config["index_bloat_bytes"] || PgHero.config["index_bloat_bytes"] || 104857600).to_i
     end
 
     def aws_access_key_id

data/lib/pghero/methods/basic.rb

@@ -18,6 +18,10 @@ module PgHero
         select_one("SELECT current_database()")
       end
 
+      def current_user
+        select_one("SELECT current_user")
+      end
+
       def server_version
         @server_version ||= select_one("SHOW server_version")
       end
@@ -38,7 +42,11 @@ module PgHero
         retries = 0
         begin
           result = conn.select_all(add_source(squish(sql)))
-          result = result.map { |row| Hash[row.map { |col, val| [col.to_sym, result.column_types[col].send(:cast_value, val)] }] }
+          if ActiveRecord::VERSION::STRING.to_f >= 6.1
+            result = result.map(&:symbolize_keys)
+          else
+            result = result.map { |row| Hash[row.map { |col, val| [col.to_sym, result.column_types[col].send(:cast_value, val)] }] }
+          end
           if filter_data
             query_columns.each do |column|
               result.each do |row|

data/lib/pghero/methods/explain.rb

@@ -6,7 +6,7 @@ module PgHero
         explanation = nil
 
         # use transaction for safety
-        with_transaction(statement_timeout: (explain_timeout_sec * 1000), rollback: true) do
+        with_transaction(statement_timeout: (explain_timeout_sec * 1000).round, rollback: true) do
           if (sql.sub(/;\z/, "").include?(";") || sql.upcase.include?("COMMIT")) && !explain_safe?
             raise ActiveRecord::StatementInvalid, "Unsafe statement"
           end

data/lib/pghero/methods/query_stats.rb

@@ -56,8 +56,46 @@ module PgHero
         true
       end
 
-      def reset_query_stats(raise_errors: false)
-        execute("SELECT pg_stat_statements_reset()")
+      # TODO scope by database in PgHero 3.0
+      # (add database: database_name to options)
+      def reset_query_stats(**options)
+        reset_instance_query_stats(**options)
+      end
+
+      # resets query stats for the entire instance
+      # it's possible to reset stats for a specific
+      # database, user or query hash in Postgres 12+
+      def reset_instance_query_stats(database: nil, user: nil, query_hash: nil, raise_errors: false)
+        if database || user || query_hash
+          raise PgHero::Error, "Requires PostgreSQL 12+" if server_version_num < 120000
+
+          if database
+            database_id = execute("SELECT oid FROM pg_database WHERE datname = #{quote(database)}").first.try(:[], "oid")
+            raise PgHero::Error, "Database not found: #{database}" unless database_id
+          else
+            database_id = 0
+          end
+
+          if user
+            user_id = execute("SELECT usesysid FROM pg_user WHERE usename = #{quote(user)}").first.try(:[], "usesysid")
+            raise PgHero::Error, "User not found: #{user}" unless user_id
+          else
+            user_id = 0
+          end
+
+          if query_hash
+            query_id = query_hash.to_i
+            # may not be needed
+            # but not intuitive that all query hashes are reset with 0
+            raise PgHero::Error, "Invalid query hash: #{query_hash}" if query_id == 0
+          else
+            query_id = 0
+          end
+
+          execute("SELECT pg_stat_statements_reset(#{quote(user_id.to_i)}, #{quote(database_id.to_i)}, #{quote(query_id.to_i)})")
+        else
+          execute("SELECT pg_stat_statements_reset()")
+        end
         true
       rescue ActiveRecord::StatementInvalid => e
         raise e if raise_errors
@@ -104,31 +142,32 @@ module PgHero
           query_stats[database_id] = query_stats(limit: 1000000, database: database_name)
         end
 
-        supports_query_hash = supports_query_hash?
+        query_stats = query_stats.select { |_, v| v.any? }
 
-        if query_stats.any? { |_, v| v.any? } && reset_query_stats(raise_errors: raise_errors)
+        # nothing to do
+        return if query_stats.empty?
+
+        # use mapping, not query stats here
+        # TODO add option for this, and make default in PgHero 3.0
+        if false # mapping.size == 1 && server_version_num >= 120000
           query_stats.each do |db_id, db_query_stats|
-            if db_query_stats.any?
-              values =
-                db_query_stats.map do |qs|
-                  [
-                    db_id,
-                    qs[:query],
-                    qs[:total_minutes] * 60 * 1000,
-                    qs[:calls],
-                    now,
-                    supports_query_hash ? qs[:query_hash] : nil,
-                    qs[:user]
-                  ]
-                end
-
-              columns = %w[database query total_time calls captured_at query_hash user]
-              insert_stats("pghero_query_stats", columns, values)
+            if reset_query_stats(database: mapping[db_id], raise_errors: raise_errors)
+              insert_query_stats(db_id, db_query_stats, now)
+            end
+          end
+        else
+          if reset_query_stats(raise_errors: raise_errors)
+            query_stats.each do |db_id, db_query_stats|
+              insert_query_stats(db_id, db_query_stats, now)
             end
           end
         end
       end
 
+      def clean_query_stats
+        PgHero::QueryStats.where(database: id).where("captured_at < ?", 14.days.ago).delete_all
+      end
+
       def slow_queries(query_stats: nil, **options)
         query_stats ||= self.query_stats(options)
         query_stats.select { |q| q[:calls].to_i >= slow_query_calls.to_i && q[:average_time].to_f >= slow_query_ms.to_f }
@@ -166,14 +205,15 @@ module PgHero
         if query_stats_enabled?
           limit ||= 100
           sort ||= "total_minutes"
+          total_time = server_version_num >= 130000 ? "(total_plan_time + total_exec_time)" : "total_time"
           query = <<-SQL
             WITH query_stats AS (
               SELECT
                 LEFT(query, 10000) AS query,
                 #{supports_query_hash? ? "queryid" : "md5(query)"} AS query_hash,
                 rolname AS user,
-                (total_time / 1000 / 60) AS total_minutes,
-                (total_time / calls) AS average_time,
+                (#{total_time} / 1000 / 60) AS total_minutes,
+                (#{total_time} / calls) AS average_time,
                 calls
               FROM
                 pg_stat_statements
@@ -182,6 +222,7 @@ module PgHero
               INNER JOIN
                 pg_roles ON pg_roles.oid = pg_stat_statements.userid
               WHERE
+                calls > 0 AND
                 pg_database.datname = #{database ? quote(database) : "current_database()"}
                 #{query_hash ? "AND queryid = #{quote(query_hash)}" : nil}
             )
@@ -285,6 +326,24 @@ module PgHero
       def normalize_query(query)
         squish(query.to_s.gsub(/\?(, ?\?)+/, "?").gsub(/\/\*.+?\*\//, ""))
       end
+
+      def insert_query_stats(db_id, db_query_stats, now)
+        values =
+          db_query_stats.map do |qs|
+            [
+              db_id,
+              qs[:query],
+              qs[:total_minutes] * 60 * 1000,
+              qs[:calls],
+              now,
+              supports_query_hash? ? qs[:query_hash] : nil,
+              qs[:user]
+            ]
+          end
+
+        columns = %w[database query total_time calls captured_at query_hash user]
+        insert_stats("pghero_query_stats", columns, values)
+      end
     end
   end
 end

data/lib/pghero/methods/space.rb

@@ -129,6 +129,10 @@ module PgHero
         insert_stats("pghero_space_stats", columns, values) if values.any?
       end
 
+      def clean_space_stats
+        PgHero::SpaceStats.where(database: id).where("captured_at < ?", 90.days.ago).delete_all
+      end
+
       def space_stats_enabled?
         table_exists?("pghero_space_stats")
       end

data/lib/pghero/methods/system.rb

@@ -5,7 +5,7 @@ module PgHero
         !system_stats_provider.nil?
       end
 
-      # TODO require AWS 2+ automatically
+      # TODO remove defined checks in 3.0
       def system_stats_provider
         if aws_db_instance_identifier && (defined?(Aws) || defined?(AWS))
           :aws
@@ -133,7 +133,7 @@ module PgHero
       private
 
       def gcp_stats(metric_name, duration: nil, period: nil, offset: nil, series: false)
-        require "google/cloud/monitoring"
+        require "google/cloud/monitoring/v3"
 
         # TODO DRY with RDS stats
         duration = (duration || 1.hour).to_i
@@ -142,30 +142,63 @@ module PgHero
         end_time = Time.at(((Time.now - offset).to_f / period).ceil * period)
         start_time = end_time - duration
 
-        client = Google::Cloud::Monitoring::Metric.new
-
-        interval = Google::Monitoring::V3::TimeInterval.new
-        interval.end_time = Google::Protobuf::Timestamp.new(seconds: end_time.to_i)
-        # subtract period to make sure we get first data point
-        interval.start_time = Google::Protobuf::Timestamp.new(seconds: (start_time - period).to_i)
-
-        aggregation = Google::Monitoring::V3::Aggregation.new
-        # may be better to use ALIGN_NEXT_OLDER for space stats to show most recent data point
-        # stick with average for now to match AWS
-        aggregation.per_series_aligner = Google::Monitoring::V3::Aggregation::Aligner::ALIGN_MEAN
-        aggregation.alignment_period = period
-
         # validate input since we need to interpolate below
         raise Error, "Invalid metric name" unless metric_name =~ /\A[a-z\/_]+\z/i
         raise Error, "Invalid database id" unless gcp_database_id =~ /\A[a-z\-:]+\z/i
 
-        results = client.list_time_series(
-          "projects/#{gcp_database_id.split(":").first}",
-          "metric.type = \"cloudsql.googleapis.com/database/#{metric_name}\" AND resource.label.database_id = \"#{gcp_database_id}\"",
-          interval,
-          Google::Monitoring::V3::ListTimeSeriesRequest::TimeSeriesView::FULL,
-          aggregation: aggregation
-        )
+        # we handle three situations:
+        # 1. google-cloud-monitoring-v3
+        # 2. google-cloud-monitoring >= 1
+        # 3. google-cloud-monitoring < 1
+
+        # for situations 1 and 2
+        # Google::Cloud::Monitoring.metric_service is documented
+        # but doesn't work for situation 1
+        if defined?(Google::Cloud::Monitoring::V3::MetricService::Client)
+          client = Google::Cloud::Monitoring::V3::MetricService::Client.new
+
+          interval = Google::Cloud::Monitoring::V3::TimeInterval.new
+          interval.end_time = Google::Protobuf::Timestamp.new(seconds: end_time.to_i)
+          # subtract period to make sure we get first data point
+          interval.start_time = Google::Protobuf::Timestamp.new(seconds: (start_time - period).to_i)
+
+          aggregation = Google::Cloud::Monitoring::V3::Aggregation.new
+          # may be better to use ALIGN_NEXT_OLDER for space stats to show most recent data point
+          # stick with average for now to match AWS
+          aggregation.per_series_aligner = Google::Cloud::Monitoring::V3::Aggregation::Aligner::ALIGN_MEAN
+          aggregation.alignment_period = period
+
+          results = client.list_time_series({
+            name: "projects/#{gcp_database_id.split(":").first}",
+            filter: "metric.type = \"cloudsql.googleapis.com/database/#{metric_name}\" AND resource.label.database_id = \"#{gcp_database_id}\"",
+            interval: interval,
+            view: Google::Cloud::Monitoring::V3::ListTimeSeriesRequest::TimeSeriesView::FULL,
+            aggregation: aggregation
+          })
+        else
+          require "google/cloud/monitoring"
+
+          client = Google::Cloud::Monitoring::Metric.new
+
+          interval = Google::Monitoring::V3::TimeInterval.new
+          interval.end_time = Google::Protobuf::Timestamp.new(seconds: end_time.to_i)
+          # subtract period to make sure we get first data point
+          interval.start_time = Google::Protobuf::Timestamp.new(seconds: (start_time - period).to_i)
+
+          aggregation = Google::Monitoring::V3::Aggregation.new
+          # may be better to use ALIGN_NEXT_OLDER for space stats to show most recent data point
+          # stick with average for now to match AWS
+          aggregation.per_series_aligner = Google::Monitoring::V3::Aggregation::Aligner::ALIGN_MEAN
+          aggregation.alignment_period = period
+
+          results = client.list_time_series(
+            "projects/#{gcp_database_id.split(":").first}",
+            "metric.type = \"cloudsql.googleapis.com/database/#{metric_name}\" AND resource.label.database_id = \"#{gcp_database_id}\"",
+            interval,
+            Google::Monitoring::V3::ListTimeSeriesRequest::TimeSeriesView::FULL,
+            aggregation: aggregation
+          )
+        end
 
         data = {}
         result = results.first