ruby-pg-extras 2.3.0 → 3.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a93ebb6cb69e2591e7e9c327b326edf9353105b985f0023ec00deceec631e398
-  data.tar.gz: 0e6b50933c072c9c04fba24875d71c0c561ecf4b5a0cb60f8cb66df9cb000075
+  metadata.gz: ed716b44ced4653bf0c794ea1690b99cecf460f30dc767a56f328748790d4c0f
+  data.tar.gz: 3652a5add3cfd162959578af4d840ee494f99311d4b2e48ffbd2f1ca5c078b1e
 SHA512:
-  metadata.gz: b855303347b32b4e20e9a9061f058843f8a337968344d3517a14a70000849691f378e077163720f487c2798df441c1a59f7f53e80b6d45ef2fe7796854cf79f6
-  data.tar.gz: 4429a4634e5b154dca182bdb232c64009178f287a00b1c58594dd089221ab2cc1520522ba2112153dd645905685d7751b7c51f9d9a87968d0ab416af93797d26
+  metadata.gz: 3078b1e69e4a36686cc76a2c5326a35b8a61b4751cfe1031df74cda6306ae62fcd07026be2218ca40ac83dd535d0748e608bf00e5e3571aae9de0f1d794d43ff
+  data.tar.gz: 116e9de1f4a0dc927eef92e6bc88f354c54a5929831d68d7a19c58efbd8085329b0abff6cde1d482757c41d27225191e50bf2852acd9d57637d2306f61568fa7

data/.circleci/config.yml

@@ -31,7 +31,8 @@ jobs:
       - checkout
       - run: gem update --system
       - run: gem install bundler
-      - run: bundle install --path vendor/bundle
+      - run: bundle config set --local path 'vendor/bundle'
+      - run: bundle install
       - run: sudo apt-get update --allow-releaseinfo-change
       - run: sudo apt install postgresql-client-11
       - run: dockerize -wait tcp://postgres11:5432 -timeout 1m

data/README.md

@@ -26,7 +26,7 @@ In your Gemfile
 gem "ruby-pg-extras"
 ```
 
-Some of the queries (e.g., `calls` and `outliers`) require [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) extension enabled.
+`calls` and `outliers` queries require [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) extension.
 
 You can check if it is enabled in your database by running:
 
@@ -39,6 +39,12 @@ You should see the similar line in the output:
 | pg_stat_statements  | 1.7  | 1.7 | track execution statistics of all SQL statements executed |
 ```
 
+`ssl_used` requires `sslinfo` extension, and `buffercache_usage`/`buffercache_usage` queries need `pg_buffercache`. You can enable them all by running:
+
+```ruby
+RubyPGExtras.add_extensions
+```
+
 ## Usage
 
 Gem expects the `ENV['DATABASE_URL']` value in the following format:
@@ -92,6 +98,18 @@ RubyPGExtras.long_running_queries(args: { threshold: "200 milliseconds" })
 
 ```
 
+## Diagnose report
+
+The simplest way to start using pg-extras is to execute a `diagnose` method. It runs a set of checks and prints out a report highlighting areas that may require additional investigation:
+
+```ruby
+RubyPGExtras.diagnose
+```
+
+![Diagnose report](https://github.com/pawurb/ruby-pg-extras/raw/master/ruby-pg-extras-diagnose.png)
+
+Keep reading to learn about methods that `diagnose` uses under the hood.
+
 ## Available methods
 
 ### `cache_hit`
@@ -236,7 +254,7 @@ This command displays all the current locks, regardless of their type.
 
 RubyPGExtras.outliers(args: { limit: 20 })
 
-                   qry                   |    exec_time     | prop_exec_time |   ncalls    | sync_io_time
+                   query                 |    exec_time     | prop_exec_time |   ncalls    | sync_io_time
 -----------------------------------------+------------------+----------------+-------------+--------------
  SELECT * FROM archivable_usage_events.. | 154:39:26.431466 | 72.2%          | 34,211,877  | 00:00:00
  COPY public.archivable_usage_events (.. | 50:38:33.198418  | 23.6%          | 13          | 13:34:21.00108
@@ -387,7 +405,7 @@ This command displays the total size of each table and materialized view in the
 
 ```ruby
 
-RubyPGExtras.unused_indexes(args: { min_scans: 20 })
+RubyPGExtras.unused_indexes(args: { max_scans: 20 })
 
           table      |                       index                | index_size | index_scans
 ---------------------+--------------------------------------------+------------+-------------

data/lib/ruby-pg-extras/diagnose_data.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require 'filesize'
+
+module RubyPGExtras
+  class DiagnoseData
+    PG_EXTRAS_TABLE_CACHE_HIT_MIN_EXPECTED = "0.985"
+    PG_EXTRAS_INDEX_CACHE_HIT_MIN_EXPECTED = "0.985"
+    PG_EXTRAS_UNUSED_INDEXES_MAX_SCANS = 20
+    PG_EXTRAS_UNUSED_INDEXES_MIN_SIZE_BYTES = Filesize.from("1 MB").to_i # 1000000 bytes
+    PG_EXTRAS_NULL_INDEXES_MIN_SIZE_MB = 1 # 1 MB
+    PG_EXTRAS_NULL_MIN_NULL_FRAC_PERCENT = 50 # 50%
+    PG_EXTRAS_BLOAT_MIN_VALUE = 10
+    PG_EXTRAS_OUTLIERS_MIN_EXEC_RATIO = 33 # 33%
+
+    def self.call
+      new.call
+    end
+
+    def call
+      {
+        table_cache_hit: table_cache_hit,
+        index_cache_hit: index_cache_hit,
+        unused_indexes: unused_indexes,
+        null_indexes: null_indexes,
+        bloat: bloat,
+        duplicate_indexes: duplicate_indexes
+      }.yield_self do |checks|
+        extensions_data = query_module.extensions(in_format: :hash)
+        pg_stats_enabled = extensions_data.find do |el|
+          el.fetch("name") == "pg_stat_statements"
+        end.fetch("installed_version", false)
+
+        ssl_info_enabled = extensions_data.find do |el|
+          el.fetch("name") == "sslinfo"
+        end.fetch("installed_version", false)
+
+        if pg_stats_enabled
+          checks.merge!({
+            outliers: outliers
+          })
+        end
+
+        if ssl_info_enabled
+          checks.merge!({
+            ssl_used: ssl_used
+          })
+        end
+
+        checks
+      end
+    end
+
+    private
+
+    def query_module
+      RubyPGExtras
+    end
+
+    def table_cache_hit
+      min_expected = ENV.fetch(
+        "PG_EXTRAS_TABLE_CACHE_HIT_MIN_EXPECTED",
+        PG_EXTRAS_TABLE_CACHE_HIT_MIN_EXPECTED
+      ).to_f
+
+      table_cache_hit_ratio = query_module.cache_hit(in_format: :hash)[1].fetch("ratio").to_f.round(6)
+
+      if table_cache_hit_ratio > min_expected
+        {
+          ok: true,
+          message: "Table cache hit ratio is correct: #{table_cache_hit_ratio}."
+        }
+      else
+        {
+          ok: false,
+          message: "Table hit ratio is too low: #{table_cache_hit_ratio}."
+        }
+      end
+    end
+
+    def index_cache_hit
+      min_expected = ENV.fetch(
+        "PG_EXTRAS_INDEX_CACHE_HIT_MIN_EXPECTED",
+        PG_EXTRAS_INDEX_CACHE_HIT_MIN_EXPECTED
+      ).to_f
+
+      index_cache_hit_ratio = query_module.cache_hit(in_format: :hash)[0].fetch("ratio").to_f.round(6)
+
+      if index_cache_hit_ratio > min_expected
+        {
+          ok: true,
+          message: "Index hit ratio is correct: #{index_cache_hit_ratio}."
+        }
+      else
+        {
+          ok: false,
+          message: "Index hit ratio is too low: #{index_cache_hit_ratio}."
+        }
+      end
+    end
+
+    def ssl_used
+      ssl_connection = query_module.ssl_used(in_format: :hash)[0].fetch("ssl_is_used")
+
+      if ssl_connection
+        {
+          ok: true,
+          message: "Database client is using a secure SSL connection."
+        }
+      else
+        {
+          ok: false,
+          message: "Database client is using an unencrypted connection."
+        }
+      end
+    end
+
+    def unused_indexes
+      indexes = query_module.unused_indexes(
+        in_format: :hash,
+        args: { min_scans: PG_EXTRAS_UNUSED_INDEXES_MAX_SCANS }
+      ).select do |i|
+        Filesize.from(i.fetch("index_size")).to_i >= PG_EXTRAS_UNUSED_INDEXES_MIN_SIZE_BYTES
+      end
+
+      if indexes.count == 0
+        {
+          ok: true,
+          message: "No unused indexes detected."
+        }
+      else
+        print_indexes = indexes.map do |i|
+          "'#{i.fetch('index')}' on '#{i.fetch('table')}' size #{i.fetch('index_size')}"
+        end.join(",\n")
+        {
+          ok: false,
+          message: "Unused indexes detected:\n#{print_indexes}"
+        }
+      end
+    end
+
+    def null_indexes
+      indexes = query_module.null_indexes(
+        in_format: :hash,
+        args: { min_relation_size_mb: PG_EXTRAS_NULL_INDEXES_MIN_SIZE_MB }
+      ).select do |i|
+        i.fetch("null_frac").gsub("%", "").to_f >= PG_EXTRAS_NULL_MIN_NULL_FRAC_PERCENT
+      end
+
+      if indexes.count == 0
+        {
+          ok: true,
+          message: "No null indexes detected."
+        }
+      else
+        print_indexes = indexes.map do |i|
+          "'#{i.fetch('index')}' size #{i.fetch('index_size')} null values fraction #{i.fetch('null_frac')}"
+        end.join(",\n")
+        {
+          ok: false,
+          message: "Null indexes detected:\n#{print_indexes}"
+        }
+      end
+    end
+
+    def bloat
+      bloat_data = query_module.bloat(in_format: :hash).select do |b|
+        b.fetch("bloat").to_f >= PG_EXTRAS_BLOAT_MIN_VALUE
+      end
+
+      if bloat_data.count == 0
+        {
+          ok: true,
+          message: "No bloat detected."
+        }
+      else
+        print_bloat = bloat_data.map do |b|
+          "'#{b.fetch('object_name')}' bloat #{b.fetch('bloat')} waste #{b.fetch('waste')}"
+        end.join(",\n")
+
+        {
+          ok: false,
+          message: "Bloat detected:\n#{print_bloat}"
+        }
+      end
+    end
+
+    def duplicate_indexes
+      indexes = query_module.duplicate_indexes(in_format: :hash)
+
+      if indexes.count == 0
+        {
+          ok: true,
+          message: "No duplicate indexes detected."
+        }
+      else
+        print_indexes = indexes.map do |i|
+          "'#{i.fetch('idx1')}' of size #{i.fetch('size')} is identical to '#{i.fetch('idx2')}'"
+        end.join(",\n")
+
+        {
+          ok: false,
+          message: "Duplicate indexes detected:\n#{print_indexes}"
+        }
+      end
+    end
+
+    def outliers
+      queries = query_module.outliers(in_format: :hash).select do |q|
+        q.fetch("prop_exec_time").gsub("%", "").to_f >= PG_EXTRAS_OUTLIERS_MIN_EXEC_RATIO
+      end
+
+      if queries.count == 0
+        {
+          ok: true,
+          message: "No queries using significant execution ratio detected."
+        }
+      else
+        print_queries = queries.map do |q|
+          "'#{q.fetch('query').slice(0, 30)}...' called #{q.fetch('ncalls')} times, using #{q.fetch('prop_exec_time')} of total exec time."
+        end.join(",\n")
+
+        {
+          ok: false,
+          message: "Queries using significant execution ratio detected:\n#{print_queries}"
+        }
+      end
+    end
+  end
+end

data/lib/ruby-pg-extras/diagnose_print.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'terminal-table'
+
+module RubyPGExtras
+  class DiagnosePrint
+    def self.call(data)
+      new.call(data)
+    end
+
+    def call(data)
+      rows = data.sort do |key|
+        key[1].fetch(:ok) ? 1 : -1
+      end.map do |el|
+        key = el[0]
+        val = el[1]
+        symbol = val.fetch(:ok) ? "√" : "x"
+        color = val.fetch(:ok) ? :green : :red
+
+        [
+          colorize("[#{symbol}] - #{key}", color),
+          colorize(val.fetch(:message), color)
+        ]
+      end
+
+      puts Terminal::Table.new(
+        title: title,
+        rows: rows
+      )
+    end
+
+    private
+
+    def title
+      "ruby-pg-extras - diagnose report"
+    end
+
+    def colorize(string, color)
+      if color == :red
+        "\e[0;31;49m#{string}\e[0m"
+      elsif color == :green
+        "\e[0;32;49m#{string}\e[0m"
+      else
+        raise "Unsupported color: #{color}"
+      end
+    end
+  end
+end

data/lib/ruby-pg-extras/queries/add_extensions.sql

@@ -0,0 +1,5 @@
+/* Configure extensions necessary for other queries to work */
+
+CREATE EXTENSION IF NOT EXISTS sslinfo;
+CREATE EXTENSION IF NOT EXISTS pg_buffercache;
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;

data/lib/ruby-pg-extras/queries/ssl_used.sql

@@ -1,4 +1,3 @@
 /* Check if SSL connection is used */
 
-CREATE EXTENSION IF NOT EXISTS sslinfo;
 SELECT ssl_is_used();

data/lib/ruby-pg-extras/queries/unused_indexes.sql

@@ -11,6 +11,6 @@ SELECT
   idx_scan as index_scans
 FROM pg_stat_user_indexes ui
 JOIN pg_index i ON ui.indexrelid = i.indexrelid
-WHERE NOT indisunique AND idx_scan < %{min_scans} AND pg_relation_size(relid) > 5 * 8192
+WHERE NOT indisunique AND idx_scan < %{max_scans} AND pg_relation_size(relid) > 5 * 8192
 ORDER BY pg_relation_size(i.indexrelid) / nullif(idx_scan, 0) DESC NULLS FIRST,
 pg_relation_size(i.indexrelid) DESC;

data/lib/ruby-pg-extras/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubyPGExtras
-  VERSION = "2.3.0"
+  VERSION = "3.0.6"
 end

data/lib/ruby-pg-extras.rb

@@ -3,13 +3,15 @@
 require 'terminal-table'
 require 'uri'
 require 'pg'
+require 'ruby-pg-extras/diagnose_data'
+require 'ruby-pg-extras/diagnose_print'
 
 module RubyPGExtras
   @@database_url = nil
   NEW_PG_STAT_STATEMENTS = "1.8"
 
   QUERIES = %i(
-    bloat blocking cache_hit db_settings
+    add_extensions bloat blocking cache_hit db_settings
     calls extensions table_cache_hit index_cache_hit
     index_size index_usage null_indexes locks all_locks
     long_running_queries mandelbrot outliers
@@ -28,7 +30,7 @@ module RubyPGExtras
     outliers_legacy: { limit: 10 },
     buffercache_stats: { limit: 10 },
     buffercache_usage: { limit: 20 },
-    unused_indexes: { min_scans: 50 },
+    unused_indexes: { max_scans: 50 },
     null_indexes: { min_relation_size_mb: 10 }
   })
 
@@ -67,6 +69,18 @@ module RubyPGExtras
     )
   end
 
+  def self.diagnose(in_format: :display_table)
+    data = RubyPGExtras::DiagnoseData.call
+
+    if in_format == :display_table
+      RubyPGExtras::DiagnosePrint.call(data)
+    elsif in_format == :hash
+      data
+    else
+      raise "Invalid 'in_format' argument!"
+    end
+  end
+
   def self.display_result(result, title:, in_format:)
     case in_format
     when :array

data/ruby-pg-extras.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.license       = "MIT"
   gem.add_dependency "pg"
+  gem.add_dependency "filesize"
   gem.add_dependency "terminal-table"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec"

data/spec/diagnose_data_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe RubyPGExtras::DiagnoseData do
+  subject(:result) do
+    RubyPGExtras::DiagnoseData.call
+  end
+
+  describe "call" do
+    context "stubbed cases" do
+      before do
+        expect(RubyPGExtras).to receive(:unused_indexes) {
+          [
+            { "table" => "public.plans", "index" => "index_plans_on_payer_id", "index_size" => "16 MB", "index_scans" => 0 },
+            { "table" => "public.feedbacks", "index" => "index_feedbacks_on_target_id", "index_size" => "80 kB", "index_scans" => 1 },
+            { "table" => "public.channels", "index" => "index_channels_on_slack_id", "index_size" => "56 MB", "index_scans" => 7}
+          ]
+        }
+
+        expect(RubyPGExtras).to receive(:null_indexes) {
+          [
+            { "oid" => 123, "index" => "index_plans_on_payer_id", "index_size" => "16 MB", "unique" => "t", "null_frac" => "00.00%", "expected_saving" => "0 kb" },
+            { "oid" => 321, "index" => "index_feedbacks_on_target_id", "index_size" => "80 kB", "unique" => "f", "null_frac" => "97.00%", "expected_saving" => "77 kb" },
+            { "oid" => 231, "index" => "index_channels_on_slack_id", "index_size" => "56 MB", "unique" => "t", "null_frac" => "49.99%", "expected_saving" => "28 MB" }
+          ]
+        }
+
+        expect(RubyPGExtras).to receive(:bloat) {
+          [
+            { "type" => "table", "schemaname" => "public", "object_name" => "bloated_table_1", "bloat" => 8, "waste" => "0 kb" },
+            { "type" => "table", "schemaname" => "public", "object_name" => "bloated_table_2", "bloat" => 8, "waste" => "77 kb" },
+            { "type" => "schemaname", "public" => "index_channels_on_slack_id", "object_name" => "bloated_index", "bloat" => 11, "waste" => "28 MB" }
+          ]
+        }
+
+        expect(RubyPGExtras).to receive(:duplicate_indexes) {
+          [
+            { "size" => "128 kb", "idx1" => "users_pkey", "idx2" => "index_users_id" }
+          ]
+        }
+
+        expect(RubyPGExtras).to receive(:outliers) {
+          [
+            { "query" => "SELECT * FROM users WHERE users.age > 20 AND users.height > 160", "exec_time" => "154:39:26.431466", "prop_exec_time" => "72.2%", "ncalls" => "34,211,877", "sync_io_time" => "00:34:19.784318" }
+          ]
+        }
+      end
+
+      it "works" do
+        expect {
+          RubyPGExtras::DiagnosePrint.call(result)
+        }.not_to raise_error
+      end
+    end
+
+    context "real database data" do
+      before do
+        expect(RubyPGExtras).to receive(:unused_indexes) {
+          [
+            { "table" => "public.plans", "index" => "index_plans_on_payer_id", "index_size" => "2 MB", "index_scans" => 0 },
+            { "table" => "public.feedbacks", "index" => "index_feedbacks_on_target_id", "index_size" => "80 kB", "index_scans" => 1 },
+            { "table" => "public.channels", "index" => "index_channels_on_slack_id", "index_size" => "1.1 MB", "index_scans" => 7}
+          ]
+        }
+
+      end
+
+      it "works" do
+        expect {
+          RubyPGExtras::DiagnosePrint.call(result)
+        }.not_to raise_error
+      end
+    end
+  end
+end

data/spec/diagnose_print_spec.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe RubyPGExtras::DiagnosePrint do
+  subject(:print_result) do
+    RubyPGExtras::DiagnosePrint.call(data)
+  end
+
+  let(:data) do
+    {
+      :table_cache_hit =>  {
+        :ok => false,
+        :message => "Table hit ratio too low: 0.906977."
+       },
+      :index_cache_hit => {
+        :ok => false,
+        :message => "Index hit ratio is too low: 0.818182."
+      },
+      :ssl_used => {
+        :ok => true,
+        :message => "Database client is using a secure SSL connection."
+      }
+    }
+  end
+
+  describe "call" do
+    it "works" do
+      expect {
+        print_result
+      }.not_to raise_error
+    end
+  end
+end

data/spec/smoke_spec.rb

@@ -3,11 +3,6 @@
 require 'spec_helper'
 
 describe RubyPGExtras do
-  before(:all) do
-    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_buffercache;")
-    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_stat_statements;")
-  end
-
   RubyPGExtras::QUERIES.each do |query_name|
     it "#{query_name} description can be read" do
       expect do

data/spec/spec_helper.rb

@@ -17,3 +17,11 @@ else
 end
 
 ENV["DATABASE_URL"] ||= "postgresql://postgres:secret@localhost:#{port}/ruby-pg-extras-test"
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_stat_statements;")
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_buffercache;")
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS sslinfo;")
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-pg-extras
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 3.0.6
 platform: ruby
 authors:
 - pawurb
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-27 00:00:00.000000000 Z
+date: 2021-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: filesize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -83,6 +97,9 @@ files:
 - Rakefile
 - docker-compose.yml.sample
 - lib/ruby-pg-extras.rb
+- lib/ruby-pg-extras/diagnose_data.rb
+- lib/ruby-pg-extras/diagnose_print.rb
+- lib/ruby-pg-extras/queries/add_extensions.sql
 - lib/ruby-pg-extras/queries/all_locks.sql
 - lib/ruby-pg-extras/queries/bloat.sql
 - lib/ruby-pg-extras/queries/blocking.sql
@@ -116,7 +133,10 @@ files:
 - lib/ruby-pg-extras/queries/unused_indexes.sql
 - lib/ruby-pg-extras/queries/vacuum_stats.sql
 - lib/ruby-pg-extras/version.rb
+- ruby-pg-extras-diagnose.png
 - ruby-pg-extras.gemspec
+- spec/diagnose_data_spec.rb
+- spec/diagnose_print_spec.rb
 - spec/smoke_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/pawurb/ruby-pg-extras
@@ -143,5 +163,7 @@ signing_key:
 specification_version: 4
 summary: Ruby PostgreSQL performance database insights
 test_files:
+- spec/diagnose_data_spec.rb
+- spec/diagnose_print_spec.rb
 - spec/smoke_spec.rb
 - spec/spec_helper.rb