pghero_fork 2.7.3

data/lib/pghero/methods/system.rb

@@ -0,0 +1,287 @@
+module PgHero
+  module Methods
+    module System
+      def system_stats_enabled?
+        !system_stats_provider.nil?
+      end
+
+      # TODO remove defined checks in 3.0
+      def system_stats_provider
+        if aws_db_instance_identifier && (defined?(Aws) || defined?(AWS))
+          :aws
+        elsif gcp_database_id
+          :gcp
+        elsif azure_resource_id
+          :azure
+        end
+      end
+
+      def cpu_usage(**options)
+        system_stats(:cpu, **options)
+      end
+
+      def connection_stats(**options)
+        system_stats(:connections, **options)
+      end
+
+      def replication_lag_stats(**options)
+        system_stats(:replication_lag, **options)
+      end
+
+      def read_iops_stats(**options)
+        system_stats(:read_iops, **options)
+      end
+
+      def write_iops_stats(**options)
+        system_stats(:write_iops, **options)
+      end
+
+      def free_space_stats(**options)
+        system_stats(:free_space, **options)
+      end
+
+      def rds_stats(metric_name, duration: nil, period: nil, offset: nil, series: false)
+        if system_stats_enabled?
+          aws_options = {region: region}
+          if access_key_id
+            aws_options[:access_key_id] = access_key_id
+            aws_options[:secret_access_key] = secret_access_key
+          end
+
+          client =
+            if defined?(Aws)
+              Aws::CloudWatch::Client.new(aws_options)
+            else
+              AWS::CloudWatch.new(aws_options).client
+            end
+
+          duration = (duration || 1.hour).to_i
+          period = (period || 1.minute).to_i
+          offset = (offset || 0).to_i
+          end_time = Time.at(((Time.now - offset).to_f / period).ceil * period)
+          start_time = end_time - duration
+
+          resp = client.get_metric_statistics(
+            namespace: "AWS/RDS",
+            metric_name: metric_name,
+            dimensions: [{name: "DBInstanceIdentifier", value: aws_db_instance_identifier}],
+            start_time: start_time.iso8601,
+            end_time: end_time.iso8601,
+            period: period,
+            statistics: ["Average"]
+          )
+          data = {}
+          resp[:datapoints].sort_by { |d| d[:timestamp] }.each do |d|
+            data[d[:timestamp]] = d[:average]
+          end
+
+          add_missing_data(data, start_time, end_time, period) if series
+
+          data
+        else
+          raise NotEnabled, "System stats not enabled"
+        end
+      end
+
+      def azure_stats(metric_name, duration: nil, period: nil, offset: nil, series: false)
+        # TODO DRY with RDS stats
+        duration = (duration || 1.hour).to_i
+        period = (period || 1.minute).to_i
+        offset = (offset || 0).to_i
+        end_time = Time.at(((Time.now - offset).to_f / period).ceil * period)
+        start_time = end_time - duration
+
+        interval =
+          case period
+          when 60
+            "PT1M"
+          when 300
+            "PT5M"
+          when 900
+            "PT15M"
+          when 1800
+            "PT30M"
+          when 3600
+            "PT1H"
+          else
+            raise Error, "Unsupported period"
+          end
+
+        client = Azure::Monitor::Profiles::Latest::Mgmt::Client.new
+        timespan = "#{start_time.iso8601}/#{end_time.iso8601}"
+        results = client.metrics.list(
+          azure_resource_id,
+          metricnames: metric_name,
+          aggregation: "Average",
+          timespan: timespan,
+          interval: interval
+        )
+
+        data = {}
+        result = results.value.first
+        if result
+          result.timeseries.first.data.each do |point|
+            data[point.time_stamp.to_time] = point.average
+          end
+        end
+
+        add_missing_data(data, start_time, end_time, period) if series
+
+        data
+      end
+
+      private
+
+      def gcp_stats(metric_name, duration: nil, period: nil, offset: nil, series: false)
+        require "google/cloud/monitoring/v3"
+
+        # TODO DRY with RDS stats
+        duration = (duration || 1.hour).to_i
+        period = (period || 1.minute).to_i
+        offset = (offset || 0).to_i
+        end_time = Time.at(((Time.now - offset).to_f / period).ceil * period)
+        start_time = end_time - duration
+
+        # validate input since we need to interpolate below
+        raise Error, "Invalid metric name" unless metric_name =~ /\A[a-z\/_]+\z/i
+        raise Error, "Invalid database id" unless gcp_database_id =~ /\A[a-z0-9\-:]+\z/i
+
+        # we handle three situations:
+        # 1. google-cloud-monitoring-v3
+        # 2. google-cloud-monitoring >= 1
+        # 3. google-cloud-monitoring < 1
+
+        # for situations 1 and 2
+        # Google::Cloud::Monitoring.metric_service is documented
+        # but doesn't work for situation 1
+        if defined?(Google::Cloud::Monitoring::V3::MetricService::Client)
+          client = Google::Cloud::Monitoring::V3::MetricService::Client.new
+
+          interval = Google::Cloud::Monitoring::V3::TimeInterval.new
+          interval.end_time = Google::Protobuf::Timestamp.new(seconds: end_time.to_i)
+          # subtract period to make sure we get first data point
+          interval.start_time = Google::Protobuf::Timestamp.new(seconds: (start_time - period).to_i)
+
+          aggregation = Google::Cloud::Monitoring::V3::Aggregation.new
+          # may be better to use ALIGN_NEXT_OLDER for space stats to show most recent data point
+          # stick with average for now to match AWS
+          aggregation.per_series_aligner = Google::Cloud::Monitoring::V3::Aggregation::Aligner::ALIGN_MEAN
+          aggregation.alignment_period = period
+
+          results = client.list_time_series({
+            name: "projects/#{gcp_database_id.split(":").first}",
+            filter: "metric.type = \"cloudsql.googleapis.com/database/#{metric_name}\" AND resource.label.database_id = \"#{gcp_database_id}\"",
+            interval: interval,
+            view: Google::Cloud::Monitoring::V3::ListTimeSeriesRequest::TimeSeriesView::FULL,
+            aggregation: aggregation
+          })
+        else
+          require "google/cloud/monitoring"
+
+          client = Google::Cloud::Monitoring::Metric.new
+
+          interval = Google::Monitoring::V3::TimeInterval.new
+          interval.end_time = Google::Protobuf::Timestamp.new(seconds: end_time.to_i)
+          # subtract period to make sure we get first data point
+          interval.start_time = Google::Protobuf::Timestamp.new(seconds: (start_time - period).to_i)
+
+          aggregation = Google::Monitoring::V3::Aggregation.new
+          # may be better to use ALIGN_NEXT_OLDER for space stats to show most recent data point
+          # stick with average for now to match AWS
+          aggregation.per_series_aligner = Google::Monitoring::V3::Aggregation::Aligner::ALIGN_MEAN
+          aggregation.alignment_period = period
+
+          results = client.list_time_series(
+            "projects/#{gcp_database_id.split(":").first}",
+            "metric.type = \"cloudsql.googleapis.com/database/#{metric_name}\" AND resource.label.database_id = \"#{gcp_database_id}\"",
+            interval,
+            Google::Monitoring::V3::ListTimeSeriesRequest::TimeSeriesView::FULL,
+            aggregation: aggregation
+          )
+        end
+
+        data = {}
+        result = results.first
+        if result
+          result.points.each do |point|
+            time = Time.at(point.interval.start_time.seconds)
+            value = point.value.double_value
+            value *= 100 if metric_name == "cpu/utilization"
+            data[time] = value
+          end
+        end
+
+        add_missing_data(data, start_time, end_time, period) if series
+
+        data
+      end
+
+      def system_stats(metric_key, **options)
+        case system_stats_provider
+        when :aws
+          metrics = {
+            cpu: "CPUUtilization",
+            connections: "DatabaseConnections",
+            replication_lag: "ReplicaLag",
+            read_iops: "ReadIOPS",
+            write_iops: "WriteIOPS",
+            free_space: "FreeStorageSpace"
+          }
+          rds_stats(metrics[metric_key], **options)
+        when :gcp
+          if metric_key == :free_space
+            quota = gcp_stats("disk/quota", **options)
+            used = gcp_stats("disk/bytes_used", **options)
+            free_space(quota, used)
+          else
+            metrics = {
+              cpu: "cpu/utilization",
+              connections: "postgresql/num_backends",
+              replication_lag: "replication/replica_lag",
+              read_iops: "disk/read_ops_count",
+              write_iops: "disk/write_ops_count"
+            }
+            gcp_stats(metrics[metric_key], **options)
+          end
+        when :azure
+          if metric_key == :free_space
+            quota = azure_stats("storage_limit", **options)
+            used = azure_stats("storage_used", **options)
+            free_space(quota, used)
+          else
+            # no read_iops, write_iops
+            # could add io_consumption_percent
+            metrics = {
+              cpu: "cpu_percent",
+              connections: "active_connections",
+              replication_lag: "pg_replica_log_delay_in_seconds"
+            }
+            raise Error, "Metric not supported" unless metrics[metric_key]
+            azure_stats(metrics[metric_key], **options)
+          end
+        else
+          raise NotEnabled, "System stats not enabled"
+        end
+      end
+
+      # only use data points included in both series
+      # this also eliminates need to align Time.now
+      def free_space(quota, used)
+        data = {}
+        quota.each do |k, v|
+          data[k] = v - used[k] if v && used[k]
+        end
+        data
+      end
+
+      def add_missing_data(data, start_time, end_time, period)
+        time = start_time
+        end_time = end_time
+        while time < end_time
+          data[time] ||= nil
+          time += period
+        end
+      end
+    end
+  end
+end

data/lib/pghero/methods/tables.rb

@@ -0,0 +1,68 @@
+module PgHero
+  module Methods
+    module Tables
+      def table_hit_rate
+        select_one(<<-SQL
+          SELECT
+            sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read), 0) AS rate
+          FROM
+            pg_statio_user_tables
+        SQL
+        )
+      end
+
+      def table_caching
+        select_all <<-SQL
+          SELECT
+            schemaname AS schema,
+            relname AS table,
+            CASE WHEN heap_blks_hit + heap_blks_read = 0 THEN
+              0
+            ELSE
+              ROUND(1.0 * heap_blks_hit / (heap_blks_hit + heap_blks_read), 2)
+            END AS hit_rate
+          FROM
+            pg_statio_user_tables
+          ORDER BY
+            2 DESC, 1
+        SQL
+      end
+
+      def unused_tables
+        select_all <<-SQL
+          SELECT
+            schemaname AS schema,
+            relname AS table,
+            n_live_tup AS estimated_rows
+          FROM
+            pg_stat_user_tables
+          WHERE
+            idx_scan = 0
+          ORDER BY
+            n_live_tup DESC,
+            relname ASC
+         SQL
+      end
+
+      def table_stats(schema: nil, table: nil)
+        select_all <<-SQL
+          SELECT
+            nspname AS schema,
+            relname AS table,
+            reltuples::bigint AS estimated_rows,
+            pg_total_relation_size(pg_class.oid) AS size_bytes
+          FROM
+            pg_class
+          INNER JOIN
+            pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+          WHERE
+            relkind = 'r'
+            #{schema ? "AND nspname = #{quote(schema)}" : nil}
+            #{table ? "AND relname IN (#{Array(table).map { |t| quote(t) }.join(", ")})" : nil}
+          ORDER BY
+            1, 2
+        SQL
+      end
+    end
+  end
+end

data/lib/pghero/methods/users.rb

@@ -0,0 +1,87 @@
+module PgHero
+  module Methods
+    module Users
+      # documented as unsafe to pass user input
+      # TODO quote in 3.0, but still not officially supported
+      def create_user(user, password: nil, schema: "public", database: nil, readonly: false, tables: nil)
+        password ||= random_password
+        database ||= PgHero.connection_config(connection_model)[:database]
+
+        commands =
+          [
+            "CREATE ROLE #{user} LOGIN PASSWORD #{quote(password)}",
+            "GRANT CONNECT ON DATABASE #{database} TO #{user}",
+            "GRANT USAGE ON SCHEMA #{schema} TO #{user}"
+          ]
+        if readonly
+          if tables
+            commands.concat table_grant_commands("SELECT", tables, user)
+          else
+            commands << "GRANT SELECT ON ALL TABLES IN SCHEMA #{schema} TO #{user}"
+            commands << "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} GRANT SELECT ON TABLES TO #{user}"
+          end
+        else
+          if tables
+            commands.concat table_grant_commands("ALL PRIVILEGES", tables, user)
+          else
+            commands << "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA #{schema} TO #{user}"
+            commands << "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA #{schema} TO #{user}"
+            commands << "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} GRANT ALL PRIVILEGES ON TABLES TO #{user}"
+            commands << "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} GRANT ALL PRIVILEGES ON SEQUENCES TO #{user}"
+          end
+        end
+
+        # run commands
+        connection_model.transaction do
+          commands.each do |command|
+            execute command
+          end
+        end
+
+        {password: password}
+      end
+
+      # documented as unsafe to pass user input
+      # TODO quote in 3.0, but still not officially supported
+      def drop_user(user, schema: "public", database: nil)
+        database ||= PgHero.connection_config(connection_model)[:database]
+
+        # thanks shiftb
+        commands =
+          [
+            "REVOKE CONNECT ON DATABASE #{database} FROM #{user}",
+            "REVOKE USAGE ON SCHEMA #{schema} FROM #{user}",
+            "REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA #{schema} FROM #{user}",
+            "REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA #{schema} FROM #{user}",
+            "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} REVOKE SELECT ON TABLES FROM #{user}",
+            "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} REVOKE SELECT ON SEQUENCES FROM #{user}",
+            "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} REVOKE ALL ON SEQUENCES FROM #{user}",
+            "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} REVOKE ALL ON TABLES FROM #{user}",
+            "DROP ROLE #{user}"
+          ]
+
+        # run commands
+        connection_model.transaction do
+          commands.each do |command|
+            execute command
+          end
+        end
+
+        true
+      end
+
+      private
+
+      def random_password
+        require "securerandom"
+        SecureRandom.base64(40).delete("+/=")[0...24]
+      end
+
+      def table_grant_commands(privilege, tables, user)
+        tables.map do |table|
+          "GRANT #{privilege} ON TABLE #{table} TO #{user}"
+        end
+      end
+    end
+  end
+end

data/lib/pghero/query_stats.rb

@@ -0,0 +1,5 @@
+module PgHero
+  class QueryStats < Stats
+    self.table_name = "pghero_query_stats"
+  end
+end