ruby-pg-extras 1.5.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5536e950d12ba0a63deee72240de7d657f7cf33e8c5bb93622a5a29c6ba4898f
-  data.tar.gz: 8fa1ecf9537e4d4b8774e5fbbd37e7f329dd3d31a37ad83a93a7c1b5ad4fa9c4
+  metadata.gz: a88ff7e79fc4b77781012f9ee52afeee2462dcc46177d6bec8d24885c9822526
+  data.tar.gz: 9befdd7dd6f98195391fe323f5fabf0d74a6bb9bb0d717f7cf74c30219f91462
 SHA512:
-  metadata.gz: 836b17e1988508f27adcd8138f3d5d5c070e93bbb7ce28ad9e8f254582092798225d3cc96d5dce827662d9fcd98e82cbf08d5ea789cb2c3d65614f9865a7d134
-  data.tar.gz: de4d8f3511a66e1d636fa47fcccf828ad3bb9a6a3e0bb0f26fcf120674c7381b0422c94a210e102b78f4b1f82e56299762bfff85916dd0ee7b5800dfac10bc8d
+  metadata.gz: 0c5df44798b60ae20ad1d8bb395cce56d92e89ddde8912d0a747fcf7e695c74ea3f9a58fa71b7d570a956d908ef3261bfc776c8feb23b21a7f61c595bc3a0855
+  data.tar.gz: d8d2ddcc09c7127ff75b8897bbef5a4ad98f057c80e836bcf8849833c0580914bfd1fd3a435cce34ef4124458dcf0fb12b3dcdb1d78b6932bd629f0ddb90a2f5

data/.circleci/config.yml

@@ -6,6 +6,22 @@ jobs:
         environment:
           DATABASE_URL: postgresql://postgres:secret@localhost:5432/ruby-pg-extras-test
       - image: circleci/postgres:11.5
+        command: postgres -c shared_preload_libraries=pg_stat_statements
+        name: postgres11
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_DB: ruby-pg-extras-test
+          POSTGRES_PASSWORD: secret
+      - image: circleci/postgres:12.7
+        command: postgres -c shared_preload_libraries=pg_stat_statements
+        name: postgres12
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_DB: ruby-pg-extras-test
+          POSTGRES_PASSWORD: secret
+      - image: circleci/postgres:13.3
+        command: postgres -c shared_preload_libraries=pg_stat_statements
+        name: postgres13
         environment:
           POSTGRES_USER: postgres
           POSTGRES_DB: ruby-pg-extras-test
@@ -16,13 +32,24 @@ jobs:
       - run: gem update --system
       - run: gem install bundler
       - run: bundle install --path vendor/bundle
-      - run: sudo apt-get update
+      - run: sudo apt-get update --allow-releaseinfo-change
       - run: sudo apt install postgresql-client-11
-      - run: dockerize -wait tcp://localhost:5432 -timeout 1m
+      - run: dockerize -wait tcp://postgres11:5432 -timeout 1m
+      - run:
+          name: Run specs for PG 11
+          environment:
+            DATABASE_URL: postgresql://postgres:secret@postgres11:5432/ruby-pg-extras-test
+          command: bundle exec rspec spec/
+      - run:
+          name: Run specs for PG 12
+          environment:
+            DATABASE_URL: postgresql://postgres:secret@postgres12:5432/ruby-pg-extras-test
+          command: bundle exec rspec spec/
       - run:
-          name: Run specs
-          command: |
-            bundle exec rspec spec/
+          name: Run specs for PG 13
+          environment:
+            DATABASE_URL: postgresql://postgres:secret@postgres13:5432/ruby-pg-extras-test
+          command: bundle exec rspec spec/
 workflows:
   version: 2
   test:

data/README.md

@@ -109,6 +109,8 @@ RubyPGExtras.cache_hit
 
 This command provides information on the efficiency of the buffer cache, for both index reads (`index hit rate`) as well as table reads (`table hit rate`). A low buffer cache hit ratio can be a sign that the Postgres instance is too small for the workload.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
 ### `index_cache_hit`
 
 ```ruby
@@ -125,6 +127,8 @@ RubyPGExtras.index_cache_hit
 
 The same as `cache_hit` with each table's indexes cache hit info displayed separately.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
 ### `table_cache_hit`
 
 ```ruby
@@ -141,6 +145,28 @@ RubyPGExtras.table_cache_hit
 
 The same as `cache_hit` with each table's cache hit info displayed seperately.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
+### `db_settings`
+
+```ruby
+
+RubyPGExtras.db_settings
+
+             name             | setting | unit |
+------------------------------+---------+------+
+ checkpoint_completion_target | 0.7     |      |
+ default_statistics_target    | 100     |      |
+ effective_cache_size         | 1350000 | 8kB  |
+ effective_io_concurrency     | 1       |      |
+(truncated results for brevity)
+
+```
+
+This method displays values for selected PostgreSQL settings. You can compare them with settings recommended by [PGTune](https://pgtune.leopard.in.ua/#/) and tweak values to improve performance.
+
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
 ### `index_usage`
 
 ```ruby
@@ -178,6 +204,8 @@ RubyPGExtras.locks
 
 This command displays queries that have taken out an exclusive lock on a relation. Exclusive locks typically prevent other operations on that relation from taking place, and can be a cause of "hung" queries that are waiting for a lock to be granted.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#deadlocks)
+
 ### `all_locks`
 
 ```ruby
@@ -210,6 +238,8 @@ This command displays statements, obtained from `pg_stat_statements`, ordered by
 
 Typically, an efficient query will have an appropriate ratio of calls to total execution time, with as little time spent on I/O as possible. Queries that have a high total execution time but low call count should be investigated to improve their performance. Queries that have a high proportion of execution time being spent on synchronous I/O should also be investigated.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#missing-indexes)
+
 ### `calls`
 
 ```ruby
@@ -230,6 +260,8 @@ RubyPGExtras.calls(args: { limit: 10 })
 
 This command is much like `pg:outliers`, but ordered by the number of times a statement has been called.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#missing-indexes)
+
 ### `blocking`
 
 ```ruby
@@ -244,6 +276,8 @@ RubyPGExtras.blocking
 
 This command displays statements that are currently holding locks that other statements are waiting to be released. This can be used in conjunction with `pg:locks` to determine which statements need to be terminated in order to resolve lock contention.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#deadlocks)
+
 ### `total_index_size`
 
 ```ruby
@@ -351,6 +385,21 @@ RubyPGExtras.unused_indexes(args: { min_scans: 20 })
 
 This command displays indexes that have < 50 scans recorded against them, and are greater than 5 pages in size, ordered by size relative to the number of index scans. This command is generally useful for eliminating indexes that are unused, which can impact write performance, as well as read performance should they occupy space in memory.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#unused-indexes)
+
+### `duplicate_indexes`
+
+```ruby
+
+RubyPGExtras.duplicate_indexes
+
+| size       |  idx1        |  idx2          |  idx3    |  idx4     |
++------------+--------------+----------------+----------+-----------+
+| 128 k      | users_pkey   | index_users_id |          |           |
+```
+
+This command displays multiple indexes that have the same set of columns, same opclass, expression and predicate - which make them equivalent. Usually it's safe to drop one of them.
+
 ### `null_indexes`
 
 ```ruby
@@ -365,7 +414,9 @@ RubyPGExtras.null_indexes(args: { min_relation_size_mb: 10 })
 
 ```
 
-This commands displays indexes that contain `NULL` values. A high ratio of `NULL` values means that using a partial index excluding them will be beneficial in case they are not used for searching. [Source and more info](https://hakibenita.com/postgresql-unused-index-size).
+This command displays indexes that contain `NULL` values. A high ratio of `NULL` values means that using a partial index excluding them will be beneficial in case they are not used for searching.
+
+[More info](https://pawelurbanek.com/postgresql-fix-performance#null-indexes)
 
 ### `seq_scans`
 
@@ -389,6 +440,8 @@ RubyPGExtras.seq_scans
 
 This command displays the number of sequential scans recorded against all tables, descending by count of sequential scans. Tables that have very high numbers of sequential scans may be under-indexed, and it may be worth investigating queries that read from these tables.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#missing-indexes)
+
 ### `long_running_queries`
 
 ```ruby
@@ -444,6 +497,8 @@ RubyPGExtras.bloat
 
 This command displays an estimation of table "bloat" – space allocated to a relation that is full of dead tuples, that has yet to be reclaimed. Tables that have a high bloat ratio, typically 10 or greater, should be investigated to see if vacuuming is aggressive enough, and can be a sign of high table churn.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#bloat)
+
 ### `vacuum_stats`
 
 ```ruby
@@ -472,6 +527,22 @@ RubyPGExtras.kill_all
 
 This commands kills all the currently active connections to the database. It can be useful as a last resort when your database is stuck in a deadlock.
 
+### `buffercache_stats`
+
+```ruby
+RubyPGExtras.buffercache_stats(args: { limit: 10 })
+```
+
+This command shows the relations buffered in database share buffer, ordered by percentage taken. It also shows that how much of the whole relation is buffered.
+
+### `buffercache_usage`
+
+```ruby
+RubyPGExtras.buffercache_usage(args: { limit: 20 })
+```
+
+This command calculates how many blocks from which table are currently cached.
+
 ### `extensions`
 
 ```ruby
@@ -491,3 +562,18 @@ RubyPGExtras.mandelbrot
 ```
 
 This command outputs the Mandelbrot set, calculated through SQL.
+
+## Testing
+
+```bash
+cp docker-compose.yml.sample docker-compose.yml
+docker compose up -d
+rake test_all
+```
+
+## Query sources
+
+- [https://github.com/heroku/heroku-pg-extras](https://github.com/heroku/heroku-pg-extras)
+- [https://hakibenita.com/postgresql-unused-index-size](https://hakibenita.com/postgresql-unused-index-size)
+- [https://sites.google.com/site/itmyshare/database-tips-and-examples/postgres/useful-sqls-to-check-contents-of-postgresql-shared_buffer](https://sites.google.com/site/itmyshare/database-tips-and-examples/postgres/useful-sqls-to-check-contents-of-postgresql-shared_buffer)
+- [https://wiki.postgresql.org/wiki/Index_Maintenance](https://wiki.postgresql.org/wiki/Index_Maintenance)

data/Rakefile

@@ -3,3 +3,7 @@ require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
+desc 'Test all PG versions'
+task :test_all do
+  system("PG_VERSION=11 bundle exec rspec spec/ && PG_VERSION=12 bundle exec rspec spec/ && PG_VERSION=13 bundle exec rspec spec/")
+end

data/docker-compose.yml.sample

@@ -1,11 +1,30 @@
 version: '3'
 
 services:
-  postgres:
+  postgres11:
     image: postgres:11.5-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements
     environment:
       POSTGRES_USER: postgres
       POSTGRES_DB: ruby-pg-extras-test
       POSTGRES_PASSWORD: secret
     ports:
       - '5432:5432'
+  postgres12:
+    image: postgres:12.7-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: ruby-pg-extras-test
+      POSTGRES_PASSWORD: secret
+    ports:
+      - '5433:5432'
+  postgres13:
+    image: postgres:13.3-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: ruby-pg-extras-test
+      POSTGRES_PASSWORD: secret
+    ports:
+      - '5434:5432'

data/lib/ruby-pg-extras/queries/buffercache_stats.sql

@@ -0,0 +1,13 @@
+/* Calculates percentages of relations buffered in database share buffer */
+
+SELECT
+  c.relname,
+  pg_size_pretty(count(*) * 8192) AS buffered,
+  round(100.0 * count(*) / (SELECT setting FROM pg_settings WHERE name = 'shared_buffers')::integer, 1) AS buffer_percent,
+  round(100.0 * count(*) * 8192 / pg_table_size(c.oid), 1) AS percent_of_relation
+FROM pg_class c
+INNER JOIN pg_buffercache b ON b.relfilenode = c.relfilenode
+INNER JOIN pg_database d ON (b.reldatabase = d.oid AND d.datname = current_database())
+GROUP BY c.oid,c.relname
+ORDER BY 3 DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/queries/buffercache_usage.sql

@@ -0,0 +1,9 @@
+/* Calculate how many blocks from which table are currently cached */
+
+SELECT c.relname, count(*) AS buffers
+FROM pg_class c
+INNER JOIN pg_buffercache b ON b.relfilenode = c.relfilenode
+INNER JOIN pg_database d ON (b.reldatabase = d.oid AND d.datname = current_database())
+GROUP BY c.relname
+ORDER BY 2 DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/queries/calls.sql

@@ -1,8 +1,8 @@
 /* Queries that have highest frequency of execution */
 
 SELECT query AS qry,
-interval '1 millisecond' * total_time AS exec_time,
-to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+interval '1 millisecond' * total_exec_time AS exec_time,
+to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
 to_char(calls, 'FM999G999G990') AS ncalls,
 interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time
 FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)

data/lib/ruby-pg-extras/queries/calls_legacy.sql

@@ -0,0 +1,9 @@
+/* Queries that have highest frequency of execution */
+
+SELECT query AS qry,
+interval '1 millisecond' * total_time AS exec_time,
+to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+to_char(calls, 'FM999G999G990') AS ncalls,
+interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time
+FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
+ORDER BY calls DESC LIMIT %{limit};

data/lib/ruby-pg-extras/queries/db_settings.sql

@@ -0,0 +1,9 @@
+/* Values of selected PostgreSQL settings */
+
+SELECT name, setting, unit, short_desc FROM pg_settings
+WHERE name IN (
+  'max_connections', 'shared_buffers', 'effective_cache_size',
+  'maintenance_work_mem', 'checkpoint_completion_target', 'wal_buffers',
+  'default_statistics_target', 'random_page_cost', 'effective_io_concurrency',
+  'work_mem', 'min_wal_size', 'max_wal_size'
+);

data/lib/ruby-pg-extras/queries/duplicate_indexes.sql

@@ -0,0 +1,11 @@
+/* Multiple indexes that have the same set of columns, same opclass, expression and predicate -- which make them equivalent. */
+
+SELECT pg_size_pretty(sum(pg_relation_size(idx))::bigint) as size,
+       (array_agg(idx))[1] as idx1, (array_agg(idx))[2] as idx2,
+       (array_agg(idx))[3] as idx3, (array_agg(idx))[4] as idx4
+FROM (
+    SELECT indexrelid::regclass as idx, (indrelid::text ||E'\n'|| indclass::text ||E'\n'|| indkey::text ||E'\n'||
+                                         coalesce(indexprs::text,'')||E'\n' || coalesce(indpred::text,'')) as key
+    FROM pg_index) sub
+GROUP BY key HAVING count(*)>1
+ORDER BY sum(pg_relation_size(idx)) DESC;

data/lib/ruby-pg-extras/queries/outliers.sql

@@ -1,10 +1,10 @@
 /* Queries that have longest execution time in aggregate */
 
-SELECT interval '1 millisecond' * total_time AS total_exec_time,
-to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+SELECT interval '1 millisecond' * total_exec_time AS total_exec_time,
+to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
 to_char(calls, 'FM999G999G999G990') AS ncalls,
 interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time,
 query AS query
 FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
-ORDER BY total_time DESC
+ORDER BY total_exec_time DESC
 LIMIT %{limit};

data/lib/ruby-pg-extras/queries/outliers_legacy.sql

@@ -0,0 +1,10 @@
+/* Queries that have longest execution time in aggregate */
+
+SELECT interval '1 millisecond' * total_time AS total_exec_time,
+to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+to_char(calls, 'FM999G999G999G990') AS ncalls,
+interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time,
+query AS query
+FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
+ORDER BY total_time DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubyPGExtras
-  VERSION = "1.5.2"
+  VERSION = "2.1.0"
 end

data/lib/ruby-pg-extras.rb

@@ -6,21 +6,27 @@ require 'pg'
 
 module RubyPGExtras
   @@database_url = nil
+  NEW_PG_STAT_STATEMENTS = "1.8"
 
   QUERIES = %i(
-    bloat blocking cache_hit
+    bloat blocking cache_hit db_settings
     calls extensions table_cache_hit index_cache_hit
     index_size index_usage null_indexes locks all_locks
     long_running_queries mandelbrot outliers
     records_rank seq_scans table_indexes_size
     table_size total_index_size total_table_size
-    unused_indexes vacuum_stats kill_all
+    unused_indexes duplicate_indexes vacuum_stats kill_all
+    buffercache_stats buffercache_usage
   )
 
   DEFAULT_ARGS = Hash.new({}).merge({
     calls: { limit: 10 },
+    calls_legacy: { limit: 10 },
     long_running_queries: { threshold: "500 milliseconds" },
     outliers: { limit: 10 },
+    outliers_legacy: { limit: 10 },
+    buffercache_stats: { limit: 10 },
+    buffercache_usage: { limit: 20 },
     unused_indexes: { min_scans: 50 },
     null_indexes: { min_relation_size_mb: 10 }
   })
@@ -36,6 +42,16 @@ module RubyPGExtras
   end
 
   def self.run_query(query_name:, in_format:, args: {})
+    if %i(calls outliers).include?(query_name)
+      pg_stat_statements_ver = RubyPGExtras.connection.exec("select installed_version from pg_available_extensions where name='pg_stat_statements'")
+        .to_a[0].fetch("installed_version", nil)
+      if pg_stat_statements_ver != nil
+        if Gem::Version.new(pg_stat_statements_ver) < Gem::Version.new(NEW_PG_STAT_STATEMENTS)
+          query_name = "#{query_name}_legacy".to_sym
+        end
+      end
+    end
+
     sql = if (custom_args = DEFAULT_ARGS[query_name].merge(args)) != {}
       sql_for(query_name: query_name) % custom_args
     else

data/spec/smoke_spec.rb

@@ -3,6 +3,11 @@
 require 'spec_helper'
 
 describe RubyPGExtras do
+  before(:all) do
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_buffercache;")
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_stat_statements;")
+  end
+
   RubyPGExtras::QUERIES.each do |query_name|
     it "#{query_name} description can be read" do
       expect do
@@ -13,9 +18,7 @@ describe RubyPGExtras do
     end
   end
 
-  PG_STATS_DEPENDENT_QUERIES = %i(calls outliers)
-
-  (RubyPGExtras::QUERIES - PG_STATS_DEPENDENT_QUERIES).each do |query_name|
+  RubyPGExtras::QUERIES.each do |query_name|
     it "#{query_name} query can be executed" do
       expect do
         RubyPGExtras.run_query(

data/spec/spec_helper.rb

@@ -4,4 +4,16 @@ require 'rubygems'
 require 'bundler/setup'
 require_relative '../lib/ruby-pg-extras'
 
-ENV["DATABASE_URL"] ||= "postgresql://postgres:secret@localhost:5432/ruby-pg-extras-test"
+pg_version = ENV["PG_VERSION"]
+
+port = if pg_version == "11"
+  "5432"
+elsif pg_version == "12"
+  "5433"
+elsif pg_version == "13"
+  "5434"
+else
+  "5432"
+end
+
+ENV["DATABASE_URL"] ||= "postgresql://postgres:secret@localhost:#{port}/ruby-pg-extras-test"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-pg-extras
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 2.1.0
 platform: ruby
 authors:
 - pawurb
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-13 00:00:00.000000000 Z
+date: 2021-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -86,8 +86,13 @@ files:
 - lib/ruby-pg-extras/queries/all_locks.sql
 - lib/ruby-pg-extras/queries/bloat.sql
 - lib/ruby-pg-extras/queries/blocking.sql
+- lib/ruby-pg-extras/queries/buffercache_stats.sql
+- lib/ruby-pg-extras/queries/buffercache_usage.sql
 - lib/ruby-pg-extras/queries/cache_hit.sql
 - lib/ruby-pg-extras/queries/calls.sql
+- lib/ruby-pg-extras/queries/calls_legacy.sql
+- lib/ruby-pg-extras/queries/db_settings.sql
+- lib/ruby-pg-extras/queries/duplicate_indexes.sql
 - lib/ruby-pg-extras/queries/extensions.sql
 - lib/ruby-pg-extras/queries/index_cache_hit.sql
 - lib/ruby-pg-extras/queries/index_size.sql
@@ -98,6 +103,7 @@ files:
 - lib/ruby-pg-extras/queries/mandelbrot.sql
 - lib/ruby-pg-extras/queries/null_indexes.sql
 - lib/ruby-pg-extras/queries/outliers.sql
+- lib/ruby-pg-extras/queries/outliers_legacy.sql
 - lib/ruby-pg-extras/queries/records_rank.sql
 - lib/ruby-pg-extras/queries/seq_scans.sql
 - lib/ruby-pg-extras/queries/table_cache_hit.sql
@@ -130,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Ruby PostgreSQL performance database insights