ruby-pg-extras 1.5.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 741b8ec49ba2e7b4933ddbe085bd3c4f90ab1640e951fdec1dfa1178c4bf8ab0
-  data.tar.gz: d1b7d405ef8da4bcd450e461a6ff4411b760128bd22e660c1dc102aaa1cf9453
+  metadata.gz: d71c264c67135e083a2072dfa8248ff46210ae386c813f23417b70c6084d01c2
+  data.tar.gz: 8749efb98e35fb17443f16a1e9a06cd8ce4e70d87f32aef4a20f21b22bfd56b9
 SHA512:
-  metadata.gz: ce5011d3e926a7b1a246b6268d7d58a8f05f26bbab22bd665209daec81f4d03867b78c8306f6d3ebd4661acc1ae96640427ea33486d3ab6f91b1716c20b11cc1
-  data.tar.gz: 1cee23d5f7e171bdd13eeea76102a59ff1ad6774a9b1c80b614e76e9d4a60d4b76b098403fd69e7e5913c001d98e0b4792fc1f58762234d0b10af7cbd6a1ef6e
+  metadata.gz: f31780c4436d533a80fd1b73c4470a09276330ef6a07b8e453807a4b12c4595c05c0dce82ef85d400d4615e9d384b6df25f4f84442007e40d187212e52b33618
+  data.tar.gz: 4de7b1452cea8ef0df3d9f6d3c4c71de7476c6cf2cedb43bdbdff8e4d5f5732890b3251938ab9a811f74e64d4b24c4d325b32a4d5511b072fe152b1b836e98ca

data/.circleci/config.yml

@@ -6,6 +6,7 @@ jobs:
         environment:
           DATABASE_URL: postgresql://postgres:secret@localhost:5432/ruby-pg-extras-test
       - image: circleci/postgres:11.5
+        command: postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c max_connections=200
         environment:
           POSTGRES_USER: postgres
           POSTGRES_DB: ruby-pg-extras-test

data/README.md

@@ -109,6 +109,8 @@ RubyPGExtras.cache_hit
 
 This command provides information on the efficiency of the buffer cache, for both index reads (`index hit rate`) as well as table reads (`table hit rate`). A low buffer cache hit ratio can be a sign that the Postgres instance is too small for the workload.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
 ### `index_cache_hit`
 
 ```ruby
@@ -125,6 +127,8 @@ RubyPGExtras.index_cache_hit
 
 The same as `cache_hit` with each table's indexes cache hit info displayed separately.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
 ### `table_cache_hit`
 
 ```ruby
@@ -141,6 +145,28 @@ RubyPGExtras.table_cache_hit
 
 The same as `cache_hit` with each table's cache hit info displayed seperately.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
+### `db_settings`
+
+```ruby
+
+RubyPGExtras.db_settings
+
+             name             | setting | unit |
+------------------------------+---------+------+
+ checkpoint_completion_target | 0.7     |      |
+ default_statistics_target    | 100     |      |
+ effective_cache_size         | 1350000 | 8kB  |
+ effective_io_concurrency     | 1       |      |
+(truncated results for brevity)
+
+```
+
+This method displays values for selected PostgreSQL settings. You can compare them with settings recommended by [PGTune](https://pgtune.leopard.in.ua/#/) and tweak values to improve performance.
+
+[More info](https://pawelurbanek.com/postgresql-fix-performance#cache-hit)
+
 ### `index_usage`
 
 ```ruby
@@ -178,6 +204,8 @@ RubyPGExtras.locks
 
 This command displays queries that have taken out an exclusive lock on a relation. Exclusive locks typically prevent other operations on that relation from taking place, and can be a cause of "hung" queries that are waiting for a lock to be granted.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#deadlocks)
+
 ### `all_locks`
 
 ```ruby
@@ -210,6 +238,8 @@ This command displays statements, obtained from `pg_stat_statements`, ordered by
 
 Typically, an efficient query will have an appropriate ratio of calls to total execution time, with as little time spent on I/O as possible. Queries that have a high total execution time but low call count should be investigated to improve their performance. Queries that have a high proportion of execution time being spent on synchronous I/O should also be investigated.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#missing-indexes)
+
 ### `calls`
 
 ```ruby
@@ -230,6 +260,8 @@ RubyPGExtras.calls(args: { limit: 10 })
 
 This command is much like `pg:outliers`, but ordered by the number of times a statement has been called.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#missing-indexes)
+
 ### `blocking`
 
 ```ruby
@@ -244,6 +276,8 @@ RubyPGExtras.blocking
 
 This command displays statements that are currently holding locks that other statements are waiting to be released. This can be used in conjunction with `pg:locks` to determine which statements need to be terminated in order to resolve lock contention.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#deadlocks)
+
 ### `total_index_size`
 
 ```ruby
@@ -351,21 +385,25 @@ RubyPGExtras.unused_indexes(args: { min_scans: 20 })
 
 This command displays indexes that have < 50 scans recorded against them, and are greater than 5 pages in size, ordered by size relative to the number of index scans. This command is generally useful for eliminating indexes that are unused, which can impact write performance, as well as read performance should they occupy space in memory.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#unused-indexes)
+
 ### `null_indexes`
 
 ```ruby
 
-RubyPGExtras.null_indexes
+RubyPGExtras.null_indexes(args: { min_relation_size_mb: 10 })
 
    oid   |         index      | index_size | unique | indexed_column | null_frac | expected_saving
 ---------+--------------------+------------+--------+----------------+-----------+-----------------
-  183764 | users_reset_token  | 1418 MB    | t      | reset_token    |   96.15%  | 1363 MB
-   88732 | plan_cancelled_at  | 1651 MB    | f      | cancelled_at   |    6.11%  | 101 MB
- 9827345 | users_email        | 22 MB      | t      | email          |   11.21%  | 2494 kB
+  183764 | users_reset_token  | 1445 MB    | t      | reset_token    |   97.00%  | 1401 MB
+   88732 | plan_cancelled_at  | 539 MB     | f      | cancelled_at   |    8.30%  | 44 MB
+ 9827345 | users_email        | 18 MB      | t      | email          |   28.67%  | 5160 kB
 
 ```
 
-This commands displays indexes that contain `NULL` values. A high ratio of `NULL` values means that using a partial index excluding them will be beneficial in case they are not used for searching. [Source and more info](https://hakibenita.com/postgresql-unused-index-size).
+This command displays indexes that contain `NULL` values. A high ratio of `NULL` values means that using a partial index excluding them will be beneficial in case they are not used for searching.
+
+[More info](https://pawelurbanek.com/postgresql-fix-performance#null-indexes)
 
 ### `seq_scans`
 
@@ -389,11 +427,13 @@ RubyPGExtras.seq_scans
 
 This command displays the number of sequential scans recorded against all tables, descending by count of sequential scans. Tables that have very high numbers of sequential scans may be under-indexed, and it may be worth investigating queries that read from these tables.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#missing-indexes)
+
 ### `long_running_queries`
 
 ```ruby
 
-RubyPGExtras.long_running_queries
+RubyPGExtras.long_running_queries(args: { threshold: "200 milliseconds" })
 
 
   pid  |    duration     |                                      query
@@ -444,6 +484,8 @@ RubyPGExtras.bloat
 
 This command displays an estimation of table "bloat" – space allocated to a relation that is full of dead tuples, that has yet to be reclaimed. Tables that have a high bloat ratio, typically 10 or greater, should be investigated to see if vacuuming is aggressive enough, and can be a sign of high table churn.
 
+[More info](https://pawelurbanek.com/postgresql-fix-performance#bloat)
+
 ### `vacuum_stats`
 
 ```ruby
@@ -472,6 +514,22 @@ RubyPGExtras.kill_all
 
 This commands kills all the currently active connections to the database. It can be useful as a last resort when your database is stuck in a deadlock.
 
+### `buffercache_stats`
+
+```ruby
+RubyPGExtras.buffercache_stats(args: { limit: 10 })
+```
+
+This command shows the relations buffered in database share buffer, ordered by percentage taken. It also shows that how much of the whole relation is buffered.
+
+### `buffercache_usage`
+
+```ruby
+RubyPGExtras.buffercache_usage(args: { limit: 20 })
+```
+
+This command calculates how many blocks from which table are currently cached.
+
 ### `extensions`
 
 ```ruby
@@ -491,3 +549,9 @@ RubyPGExtras.mandelbrot
 ```
 
 This command outputs the Mandelbrot set, calculated through SQL.
+
+## Query sources
+
+- [https://github.com/heroku/heroku-pg-extras](https://github.com/heroku/heroku-pg-extras)
+- [https://hakibenita.com/postgresql-unused-index-size](https://hakibenita.com/postgresql-unused-index-size)
+- [https://sites.google.com/site/itmyshare/database-tips-and-examples/postgres/useful-sqls-to-check-contents-of-postgresql-shared_buffer](https://sites.google.com/site/itmyshare/database-tips-and-examples/postgres/useful-sqls-to-check-contents-of-postgresql-shared_buffer)

data/docker-compose.yml.sample

@@ -1,11 +1,30 @@
 version: '3'
 
 services:
-  postgres:
+  postgres11:
     image: postgres:11.5-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c max_connections=200
     environment:
       POSTGRES_USER: postgres
       POSTGRES_DB: ruby-pg-extras-test
       POSTGRES_PASSWORD: secret
     ports:
       - '5432:5432'
+  postgres12:
+    image: postgres:12.7-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c max_connections=200
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: ruby-pg-extras-test
+      POSTGRES_PASSWORD: secret
+    ports:
+      - '5433:5432'
+  postgres13:
+    image: postgres:13.3-alpine
+    command: postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c max_connections=200
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: ruby-pg-extras-test
+      POSTGRES_PASSWORD: secret
+    ports:
+      - '5434:5432'

data/lib/ruby-pg-extras.rb

@@ -6,22 +6,29 @@ require 'pg'
 
 module RubyPGExtras
   @@database_url = nil
+  NEW_PG_STAT_STATEMENTS = "1.8"
 
   QUERIES = %i(
-    bloat blocking cache_hit
+    bloat blocking cache_hit db_settings
     calls extensions table_cache_hit index_cache_hit
     index_size index_usage null_indexes locks all_locks
     long_running_queries mandelbrot outliers
     records_rank seq_scans table_indexes_size
     table_size total_index_size total_table_size
     unused_indexes vacuum_stats kill_all
+    buffercache_stats buffercache_usage
   )
 
   DEFAULT_ARGS = Hash.new({}).merge({
     calls: { limit: 10 },
+    calls_legacy: { limit: 10 },
     long_running_queries: { threshold: "500 milliseconds" },
     outliers: { limit: 10 },
-    unused_indexes: { min_scans: 50 }
+    outliers_legacy: { limit: 10 },
+    buffercache_stats: { limit: 10 },
+    buffercache_usage: { limit: 20 },
+    unused_indexes: { min_scans: 50 },
+    null_indexes: { min_relation_size_mb: 10 }
   })
 
   QUERIES.each do |query_name|
@@ -35,6 +42,16 @@ module RubyPGExtras
   end
 
   def self.run_query(query_name:, in_format:, args: {})
+    if %i(calls outliers).include?(query_name)
+      pg_stat_statements_ver = RubyPGExtras.connection.exec("select installed_version from pg_available_extensions where name='pg_stat_statements'")
+        .to_a[0].fetch("installed_version", nil)
+      if pg_stat_statements_ver != nil
+        if Gem::Version.new(pg_stat_statements_ver) < Gem::Version.new(NEW_PG_STAT_STATEMENTS)
+          query_name = "#{query_name}_legacy".to_sym
+        end
+      end
+    end
+
     sql = if (custom_args = DEFAULT_ARGS[query_name].merge(args)) != {}
       sql_for(query_name: query_name) % custom_args
     else

data/lib/ruby-pg-extras/queries/buffercache_stats.sql

@@ -0,0 +1,13 @@
+/* Calculates percentages of relations buffered in database share buffer */
+
+SELECT
+  c.relname,
+  pg_size_pretty(count(*) * 8192) AS buffered,
+  round(100.0 * count(*) / (SELECT setting FROM pg_settings WHERE name = 'shared_buffers')::integer, 1) AS buffer_percent,
+  round(100.0 * count(*) * 8192 / pg_table_size(c.oid), 1) AS percent_of_relation
+FROM pg_class c
+INNER JOIN pg_buffercache b ON b.relfilenode = c.relfilenode
+INNER JOIN pg_database d ON (b.reldatabase = d.oid AND d.datname = current_database())
+GROUP BY c.oid,c.relname
+ORDER BY 3 DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/queries/buffercache_usage.sql

@@ -0,0 +1,9 @@
+/* Calculate how many blocks from which table are currently cached */
+
+SELECT c.relname, count(*) AS buffers
+FROM pg_class c
+INNER JOIN pg_buffercache b ON b.relfilenode = c.relfilenode
+INNER JOIN pg_database d ON (b.reldatabase = d.oid AND d.datname = current_database())
+GROUP BY c.relname
+ORDER BY 2 DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/queries/calls.sql

@@ -1,8 +1,8 @@
 /* Queries that have highest frequency of execution */
 
 SELECT query AS qry,
-interval '1 millisecond' * total_time AS exec_time,
-to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+interval '1 millisecond' * total_exec_time AS exec_time,
+to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
 to_char(calls, 'FM999G999G990') AS ncalls,
 interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time
 FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)

data/lib/ruby-pg-extras/queries/calls_legacy.sql

@@ -0,0 +1,9 @@
+/* Queries that have highest frequency of execution */
+
+SELECT query AS qry,
+interval '1 millisecond' * total_time AS exec_time,
+to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+to_char(calls, 'FM999G999G990') AS ncalls,
+interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time
+FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
+ORDER BY calls DESC LIMIT %{limit};

data/lib/ruby-pg-extras/queries/db_settings.sql

@@ -0,0 +1,9 @@
+/* Values of selected PostgreSQL settings */
+
+SELECT name, setting, unit, short_desc FROM pg_settings
+WHERE name IN (
+  'max_connections', 'shared_buffers', 'effective_cache_size',
+  'maintenance_work_mem', 'checkpoint_completion_target', 'wal_buffers',
+  'default_statistics_target', 'random_page_cost', 'effective_io_concurrency',
+  'work_mem', 'min_wal_size', 'max_wal_size'
+);

data/lib/ruby-pg-extras/queries/null_indexes.sql

@@ -1,4 +1,5 @@
-/* Find indexed columns with high null_frac */
+/* Find indexes with a high ratio of NULL values */
+
 SELECT
     c.oid,
     c.relname AS index,
@@ -7,7 +8,7 @@ SELECT
     a.attname AS indexed_column,
     CASE s.null_frac
         WHEN 0 THEN ''
-        ELSE to_char(s.null_frac * 100, '999.00%')
+        ELSE to_char(s.null_frac * 100, '999.00%%')
     END AS null_frac,
     pg_size_pretty((pg_relation_size(c.oid) * s.null_frac)::bigint) AS expected_saving
 FROM
@@ -26,7 +27,7 @@ WHERE
     AND array_length(i.indkey, 1) = 1
     -- Exclude indexes without null_frac ratio
     AND coalesce(s.null_frac, 0) != 0
-    -- Larger than 10MB
-    AND pg_relation_size(c.oid) > 10 * 1024 ^ 2
+    -- Larger than threshold
+    AND pg_relation_size(c.oid) > %{min_relation_size_mb} * 1024 ^ 2
 ORDER BY
   pg_relation_size(c.oid) * s.null_frac DESC;

data/lib/ruby-pg-extras/queries/outliers.sql

@@ -1,10 +1,10 @@
 /* Queries that have longest execution time in aggregate */
 
-SELECT interval '1 millisecond' * total_time AS total_exec_time,
-to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+SELECT interval '1 millisecond' * total_exec_time AS total_exec_time,
+to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
 to_char(calls, 'FM999G999G999G990') AS ncalls,
 interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time,
 query AS query
 FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
-ORDER BY total_time DESC
+ORDER BY total_exec_time DESC
 LIMIT %{limit};

data/lib/ruby-pg-extras/queries/outliers_legacy.sql

@@ -0,0 +1,10 @@
+/* Queries that have longest execution time in aggregate */
+
+SELECT interval '1 millisecond' * total_time AS total_exec_time,
+to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
+to_char(calls, 'FM999G999G999G990') AS ncalls,
+interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time,
+query AS query
+FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
+ORDER BY total_time DESC
+LIMIT %{limit};

data/lib/ruby-pg-extras/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubyPGExtras
-  VERSION = "1.5.0"
+  VERSION = "2.0.0"
 end

data/spec/smoke_spec.rb

@@ -3,6 +3,11 @@
 require 'spec_helper'
 
 describe RubyPGExtras do
+  before(:all) do
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_buffercache;")
+    RubyPGExtras.connection.exec("CREATE EXTENSION IF NOT EXISTS pg_stat_statements;")
+  end
+
   RubyPGExtras::QUERIES.each do |query_name|
     it "#{query_name} description can be read" do
       expect do
@@ -13,9 +18,7 @@ describe RubyPGExtras do
     end
   end
 
-  PG_STATS_DEPENDENT_QUERIES = %i(calls outliers)
-
-  (RubyPGExtras::QUERIES - PG_STATS_DEPENDENT_QUERIES).each do |query_name|
+  RubyPGExtras::QUERIES.each do |query_name|
     it "#{query_name} query can be executed" do
       expect do
         RubyPGExtras.run_query(

data/spec/spec_helper.rb

@@ -4,4 +4,16 @@ require 'rubygems'
 require 'bundler/setup'
 require_relative '../lib/ruby-pg-extras'
 
-ENV["DATABASE_URL"] ||= "postgresql://postgres:secret@localhost:5432/ruby-pg-extras-test"
+pg_version = ENV["PG_VERSION"]
+
+port = if pg_version == "11"
+  "5432"
+elsif pg_version == "12"
+  "5433"
+elsif pg_version == "13"
+  "5434"
+else
+  "5432"
+end
+
+ENV["DATABASE_URL"] ||= "postgresql://postgres:secret@localhost:#{port}/ruby-pg-extras-test"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-pg-extras
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 2.0.0
 platform: ruby
 authors:
 - pawurb
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-13 00:00:00.000000000 Z
+date: 2021-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -86,8 +86,12 @@ files:
 - lib/ruby-pg-extras/queries/all_locks.sql
 - lib/ruby-pg-extras/queries/bloat.sql
 - lib/ruby-pg-extras/queries/blocking.sql
+- lib/ruby-pg-extras/queries/buffercache_stats.sql
+- lib/ruby-pg-extras/queries/buffercache_usage.sql
 - lib/ruby-pg-extras/queries/cache_hit.sql
 - lib/ruby-pg-extras/queries/calls.sql
+- lib/ruby-pg-extras/queries/calls_legacy.sql
+- lib/ruby-pg-extras/queries/db_settings.sql
 - lib/ruby-pg-extras/queries/extensions.sql
 - lib/ruby-pg-extras/queries/index_cache_hit.sql
 - lib/ruby-pg-extras/queries/index_size.sql
@@ -98,6 +102,7 @@ files:
 - lib/ruby-pg-extras/queries/mandelbrot.sql
 - lib/ruby-pg-extras/queries/null_indexes.sql
 - lib/ruby-pg-extras/queries/outliers.sql
+- lib/ruby-pg-extras/queries/outliers_legacy.sql
 - lib/ruby-pg-extras/queries/records_rank.sql
 - lib/ruby-pg-extras/queries/seq_scans.sql
 - lib/ruby-pg-extras/queries/table_cache_hit.sql
@@ -130,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Ruby PostgreSQL performance database insights