ruby-pg-extras 1.5.3 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3de471f9f4d008d006000667b4b52d4e0d8aec0c5dc9ef55ce6045e5e8f96994
-  data.tar.gz: bb78196a09f0af1d7f4be623f1794223b3db074d19eb439e59670b8e960d06e1
+  metadata.gz: 67ee92c7a3fefa174bf843ee90a6d0247968726dbddfa57d83e02bec80614b3c
+  data.tar.gz: '0893af856168f741cfe505c39415211b079202684e4472f38614f76bc6bfc7e2'
 SHA512:
-  metadata.gz: 7ed2d3f75a540b6e8b22db825a1479a53649ee3ddde59ca7783301f5e42d89f8a8c1d40e11b42a5f1340f3b72d866531a6878c57ccfb211f43ca00d21d82c177
-  data.tar.gz: 5f6618ededb729f9d085cfdc5c5c722065ccd8beee6cb191bdfc3e6e58700fa2468d0f5aa3f7829fc9d520d7fb66fe248d897d989fe7fec9c21896cb2234bdad
+  metadata.gz: b63de82209cb9b52710c29360025058e88987ec14d48c5d59ab16f677512fab8d11f7042cd168358971b30421edf0910d59a69d082dc024440991cac5820ddf7
+  data.tar.gz: 4c9bd4431a96091ba6e5ba581ae7b7493a3b49d5329e0cd85903f90dbc13477e2d519a719ff078117b9e03a0b7dc89efb263eaafc3e458a4ff5ff6beb5e6fe23

data/.circleci/config.yml

@@ -6,6 +6,22 @@ jobs:
         environment:
           DATABASE_URL: postgresql://postgres:secret@localhost:5432/ruby-pg-extras-test
       - image: circleci/postgres:11.5
+        command: postgres -c shared_preload_libraries=pg_stat_statements
+        name: postgres11
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_DB: ruby-pg-extras-test
+          POSTGRES_PASSWORD: secret
+      - image: circleci/postgres:12.7
+        command: postgres -c shared_preload_libraries=pg_stat_statements
+        name: postgres12
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_DB: ruby-pg-extras-test
+          POSTGRES_PASSWORD: secret
+      - image: circleci/postgres:13.3
+        command: postgres -c shared_preload_libraries=pg_stat_statements
+        name: postgres13
         environment:
           POSTGRES_USER: postgres
           POSTGRES_DB: ruby-pg-extras-test
@@ -16,13 +32,24 @@ jobs:
       - run: gem update --system
       - run: gem install bundler
       - run: bundle install --path vendor/bundle
-      - run: sudo apt-get update
+      - run: sudo apt-get update --allow-releaseinfo-change
       - run: sudo apt install postgresql-client-11
-      - run: dockerize -wait tcp://localhost:5432 -timeout 1m
+      - run: dockerize -wait tcp://postgres11:5432 -timeout 1m
+      - run:
+          name: Run specs for PG 11
+          environment:
+            DATABASE_URL: postgresql://postgres:secret@postgres11:5432/ruby-pg-extras-test
+          command: bundle exec rspec spec/
+      - run:
+          name: Run specs for PG 12
+          environment:
+            DATABASE_URL: postgresql://postgres:secret@postgres12:5432/ruby-pg-extras-test
+          command: bundle exec rspec spec/
       - run:
-          name: Run specs
-          command: |
-            bundle exec rspec spec/
+          name: Run specs for PG 13
+          environment:
+            DATABASE_URL: postgresql://postgres:secret@postgres13:5432/ruby-pg-extras-test
+          command: bundle exec rspec spec/
 workflows:
   version: 2
   test:

data/README.md

@@ -165,6 +165,22 @@ RubyPGExtras.db_settings
 
 This method displays values for selected PostgreSQL settings. You can compare them with settings recommended by [PGTune](https://pgtune.leopard.in.ua/#/) and tweak values to improve performance.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
+### 'ssl_used'
+
+```ruby
+
+RubyPGExtras.ssl_used
+
+| ssl_is_used                     |
++---------------------------------+
+| t                               |
+
+```
+
+Returns boolean indicating if an encrypted SSL is currently used. Connecting to the database via an unencrypted connection is a critical security risk.
+
 ### `index_usage`
 
 ```ruby
@@ -385,6 +401,19 @@ This command displays indexes that have < 50 scans recorded against them, and ar
 
 [More info](https://pawelurbanek.com/postgresql-fix-performance#unused-indexes)
 
+### `duplicate_indexes`
+
+```ruby
+
+RubyPGExtras.duplicate_indexes
+
+| size       |  idx1        |  idx2          |  idx3    |  idx4     |
++------------+--------------+----------------+----------+-----------+
+| 128 k      | users_pkey   | index_users_id |          |           |
+```
+
+This command displays multiple indexes that have the same set of columns, same opclass, expression and predicate - which make them equivalent. Usually it's safe to drop one of them.
+
 ### `null_indexes`
 
 ```ruby
@@ -512,6 +541,22 @@ RubyPGExtras.kill_all
 
 This commands kills all the currently active connections to the database. It can be useful as a last resort when your database is stuck in a deadlock.
 
+### `buffercache_stats`
+
+```ruby
+RubyPGExtras.buffercache_stats(args: { limit: 10 })
+```
+
+This command shows the relations buffered in database share buffer, ordered by percentage taken. It also shows that how much of the whole relation is buffered.
+
+### `buffercache_usage`
+
+```ruby
+RubyPGExtras.buffercache_usage(args: { limit: 20 })
+```
+
+This command calculates how many blocks from which table are currently cached.
+
 ### `extensions`
 
 ```ruby
@@ -531,3 +576,18 @@ RubyPGExtras.mandelbrot
 ```
 
 This command outputs the Mandelbrot set, calculated through SQL.
+
+## Testing
+
+```bash
+cp docker-compose.yml.sample docker-compose.yml
+docker compose up -d
+rake test_all
+```
+
+## Query sources
+
+- [https://github.com/heroku/heroku-pg-extras](https://github.com/heroku/heroku-pg-extras)
+- [https://hakibenita.com/postgresql-unused-index-size](https://hakibenita.com/postgresql-unused-index-size)
+- [https://sites.google.com/site/itmyshare/database-tips-and-examples/postgres/useful-sqls-to-check-contents-of-postgresql-shared_buffer](https://sites.google.com/site/itmyshare/database-tips-and-examples/postgres/useful-sqls-to-check-contents-of-postgresql-shared_buffer)
+- [https://wiki.postgresql.org/wiki/Index_Maintenance](https://wiki.postgresql.org/wiki/Index_Maintenance)

data/Rakefile

@@ -3,3 +3,7 @@ require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
+desc 'Test all PG versions'
+task :test_all do
+  system("PG_VERSION=11 bundle exec rspec spec/ && PG_VERSION=12 bundle exec rspec spec/ && PG_VERSION=13 bundle exec rspec spec/")
+end

data/docker-compose.yml.sample

@@ -1,11 +1,30 @@
 version: '3'
 
 services:
-  postgres:
+  postgres11:
     image: postgres:11.5-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements
     environment:
       POSTGRES_USER: postgres
       POSTGRES_DB: ruby-pg-extras-test
       POSTGRES_PASSWORD: secret
     ports:
       - '5432:5432'
+  postgres12:
+    image: postgres:12.7-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: ruby-pg-extras-test
+      POSTGRES_PASSWORD: secret
+    ports:
+      - '5433:5432'
+  postgres13:
+    image: postgres:13.3-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: ruby-pg-extras-test
+      POSTGRES_PASSWORD: secret
+    ports:
+      - '5434:5432'

data/lib/ruby-pg-extras/queries/buffercache_stats.sql

@@ -0,0 +1,13 @@
+/* Calculates percentages of relations buffered in database share buffer */
+
+SELECT
+  c.relname,
+  pg_size_pretty(count(*) * 8192) AS buffered,
+  round(100.0 * count(*) / (SELECT setting FROM pg_settings WHERE name = 'shared_buffers')::integer, 1) AS buffer_percent,
+  round(100.0 * count(*) * 8192 / pg_table_size(c.oid), 1) AS percent_of_relation
+FROM pg_class c
+INNER JOIN pg_buffercache b ON b.relfilenode = c.relfilenode
+INNER JOIN pg_database d ON (b.reldatabase = d.oid AND d.datname = current_database())
+GROUP BY c.oid,c.relname
+ORDER BY 3 DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/queries/buffercache_usage.sql

@@ -0,0 +1,9 @@
+/* Calculate how many blocks from which table are currently cached */
+
+SELECT c.relname, count(*) AS buffers
+FROM pg_class c
+INNER JOIN pg_buffercache b ON b.relfilenode = c.relfilenode
+INNER JOIN pg_database d ON (b.reldatabase = d.oid AND d.datname = current_database())
+GROUP BY c.relname
+ORDER BY 2 DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/queries/calls.sql

@@ -1,8 +1,8 @@
 /* Queries that have highest frequency of execution */
 
 SELECT query AS qry,
-interval '1 millisecond' * total_time AS exec_time,
-to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+interval '1 millisecond' * total_exec_time AS exec_time,
+to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
 to_char(calls, 'FM999G999G990') AS ncalls,
 interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time
 FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)

data/lib/ruby-pg-extras/queries/calls_legacy.sql

@@ -0,0 +1,9 @@
+/* Queries that have highest frequency of execution */
+
+SELECT query AS qry,
+interval '1 millisecond' * total_time AS exec_time,
+to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+to_char(calls, 'FM999G999G990') AS ncalls,
+interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time
+FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
+ORDER BY calls DESC LIMIT %{limit};

data/lib/ruby-pg-extras/queries/duplicate_indexes.sql

@@ -0,0 +1,11 @@
+/* Multiple indexes that have the same set of columns, same opclass, expression and predicate. */
+
+SELECT pg_size_pretty(sum(pg_relation_size(idx))::bigint) as size,
+       (array_agg(idx))[1] as idx1, (array_agg(idx))[2] as idx2,
+       (array_agg(idx))[3] as idx3, (array_agg(idx))[4] as idx4
+FROM (
+    SELECT indexrelid::regclass as idx, (indrelid::text ||E'\n'|| indclass::text ||E'\n'|| indkey::text ||E'\n'||
+                                         coalesce(indexprs::text,'')||E'\n' || coalesce(indpred::text,'')) as key
+    FROM pg_index) sub
+GROUP BY key HAVING count(*)>1
+ORDER BY sum(pg_relation_size(idx)) DESC;

data/lib/ruby-pg-extras/queries/outliers.sql

@@ -1,10 +1,10 @@
 /* Queries that have longest execution time in aggregate */
 
-SELECT interval '1 millisecond' * total_time AS total_exec_time,
-to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+SELECT interval '1 millisecond' * total_exec_time AS total_exec_time,
+to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
 to_char(calls, 'FM999G999G999G990') AS ncalls,
 interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time,
 query AS query
 FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
-ORDER BY total_time DESC
+ORDER BY total_exec_time DESC
 LIMIT %{limit};

data/lib/ruby-pg-extras/queries/outliers_legacy.sql

@@ -0,0 +1,10 @@
+/* Queries that have longest execution time in aggregate */
+
+SELECT interval '1 millisecond' * total_time AS total_exec_time,
+to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+to_char(calls, 'FM999G999G999G990') AS ncalls,
+interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time,
+query AS query
+FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
+ORDER BY total_time DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/queries/ssl_used.sql

@@ -0,0 +1,4 @@
+/* Check if SSL connection is used */
+
+CREATE EXTENSION IF NOT EXISTS sslinfo;
+SELECT ssl_is_used();

data/lib/ruby-pg-extras/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubyPGExtras
-  VERSION = "1.5.3"
+  VERSION = "2.2.0"
 end

data/lib/ruby-pg-extras.rb

@@ -6,6 +6,7 @@ require 'pg'
 
 module RubyPGExtras
   @@database_url = nil
+  NEW_PG_STAT_STATEMENTS = "1.8"
 
   QUERIES = %i(
     bloat blocking cache_hit db_settings
@@ -14,13 +15,18 @@ module RubyPGExtras
     long_running_queries mandelbrot outliers
     records_rank seq_scans table_indexes_size
     table_size total_index_size total_table_size
-    unused_indexes vacuum_stats kill_all
+    unused_indexes duplicate_indexes vacuum_stats kill_all
+    buffercache_stats buffercache_usage ssl_used
   )
 
   DEFAULT_ARGS = Hash.new({}).merge({
     calls: { limit: 10 },
+    calls_legacy: { limit: 10 },
     long_running_queries: { threshold: "500 milliseconds" },
     outliers: { limit: 10 },
+    outliers_legacy: { limit: 10 },
+    buffercache_stats: { limit: 10 },
+    buffercache_usage: { limit: 20 },
     unused_indexes: { min_scans: 50 },
     null_indexes: { min_relation_size_mb: 10 }
   })
@@ -36,6 +42,16 @@ module RubyPGExtras
   end
 
   def self.run_query(query_name:, in_format:, args: {})
+    if %i(calls outliers).include?(query_name)
+      pg_stat_statements_ver = RubyPGExtras.connection.exec("select installed_version from pg_available_extensions where name='pg_stat_statements'")
+        .to_a[0].fetch("installed_version", nil)
+      if pg_stat_statements_ver != nil
+        if Gem::Version.new(pg_stat_statements_ver) < Gem::Version.new(NEW_PG_STAT_STATEMENTS)
+          query_name = "#{query_name}_legacy".to_sym
+        end
+      end
+    end
+
     sql = if (custom_args = DEFAULT_ARGS[query_name].merge(args)) != {}
       sql_for(query_name: query_name) % custom_args
     else

data/spec/smoke_spec.rb

@@ -3,6 +3,11 @@
 require 'spec_helper'
 
 describe RubyPGExtras do
+  before(:all) do
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_buffercache;")
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_stat_statements;")
+  end
+
   RubyPGExtras::QUERIES.each do |query_name|
     it "#{query_name} description can be read" do
       expect do
@@ -13,9 +18,7 @@ describe RubyPGExtras do
     end
   end
 
-  PG_STATS_DEPENDENT_QUERIES = %i(calls outliers)
-
-  (RubyPGExtras::QUERIES - PG_STATS_DEPENDENT_QUERIES).each do |query_name|
+  RubyPGExtras::QUERIES.each do |query_name|
     it "#{query_name} query can be executed" do
       expect do
         RubyPGExtras.run_query(

data/spec/spec_helper.rb

@@ -4,4 +4,16 @@ require 'rubygems'
 require 'bundler/setup'
 require_relative '../lib/ruby-pg-extras'
 
-ENV["DATABASE_URL"] ||= "postgresql://postgres:secret@localhost:5432/ruby-pg-extras-test"
+pg_version = ENV["PG_VERSION"]
+
+port = if pg_version == "11"
+  "5432"
+elsif pg_version == "12"
+  "5433"
+elsif pg_version == "13"
+  "5434"
+else
+  "5432"
+end
+
+ENV["DATABASE_URL"] ||= "postgresql://postgres:secret@localhost:#{port}/ruby-pg-extras-test"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-pg-extras
 version: !ruby/object:Gem::Version
-  version: 1.5.3
+  version: 2.2.0
 platform: ruby
 authors:
 - pawurb
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-16 00:00:00.000000000 Z
+date: 2021-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -86,9 +86,13 @@ files:
 - lib/ruby-pg-extras/queries/all_locks.sql
 - lib/ruby-pg-extras/queries/bloat.sql
 - lib/ruby-pg-extras/queries/blocking.sql
+- lib/ruby-pg-extras/queries/buffercache_stats.sql
+- lib/ruby-pg-extras/queries/buffercache_usage.sql
 - lib/ruby-pg-extras/queries/cache_hit.sql
 - lib/ruby-pg-extras/queries/calls.sql
+- lib/ruby-pg-extras/queries/calls_legacy.sql
 - lib/ruby-pg-extras/queries/db_settings.sql
+- lib/ruby-pg-extras/queries/duplicate_indexes.sql
 - lib/ruby-pg-extras/queries/extensions.sql
 - lib/ruby-pg-extras/queries/index_cache_hit.sql
 - lib/ruby-pg-extras/queries/index_size.sql
@@ -99,8 +103,10 @@ files:
 - lib/ruby-pg-extras/queries/mandelbrot.sql
 - lib/ruby-pg-extras/queries/null_indexes.sql
 - lib/ruby-pg-extras/queries/outliers.sql
+- lib/ruby-pg-extras/queries/outliers_legacy.sql
 - lib/ruby-pg-extras/queries/records_rank.sql
 - lib/ruby-pg-extras/queries/seq_scans.sql
+- lib/ruby-pg-extras/queries/ssl_used.sql
 - lib/ruby-pg-extras/queries/table_cache_hit.sql
 - lib/ruby-pg-extras/queries/table_indexes_size.sql
 - lib/ruby-pg-extras/queries/table_size.sql
@@ -131,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Ruby PostgreSQL performance database insights