ruby-pg-extras 1.2.3 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 390b7ac85682a6c91029781a9cf2e639993048c0c2b9b80a93e0b83ef52ba457
-  data.tar.gz: d84e02a04eaba3ef8d899cfd137139d5bb9d00310480fe0d7a2b70786538a14d
+  metadata.gz: 741b8ec49ba2e7b4933ddbe085bd3c4f90ab1640e951fdec1dfa1178c4bf8ab0
+  data.tar.gz: d1b7d405ef8da4bcd450e461a6ff4411b760128bd22e660c1dc102aaa1cf9453
 SHA512:
-  metadata.gz: e07b9007849b685743727d590ed095c68868c364a5090f2a030a50bafcfa25b15132f3954d3daa301506ec82210b4a2e2e78ddf21b92bfbcb8643bf8825f38e9
-  data.tar.gz: eda59a08e3ac273743897ab2460a1653ae4c33d1d4ec1b38e38f8a6f7cc44275f984f5231703435191b660181e23c5d75469d9fec498c73f67b3a9a70921ddad
+  metadata.gz: ce5011d3e926a7b1a246b6268d7d58a8f05f26bbab22bd665209daec81f4d03867b78c8306f6d3ebd4661acc1ae96640427ea33486d3ab6f91b1716c20b11cc1
+  data.tar.gz: 1cee23d5f7e171bdd13eeea76102a59ff1ad6774a9b1c80b614e76e9d4a60d4b76b098403fd69e7e5913c001d98e0b4792fc1f58762234d0b10af7cbd6a1ef6e

data/.circleci/config.yml

@@ -16,7 +16,8 @@ jobs:
       - run: gem update --system
       - run: gem install bundler
       - run: bundle install --path vendor/bundle
-      - run: sudo apt install postgresql-client
+      - run: sudo apt-get update
+      - run: sudo apt install postgresql-client-11
       - run: dockerize -wait tcp://localhost:5432 -timeout 1m
       - run:
           name: Run specs

data/README.md

@@ -16,6 +16,8 @@ Alternative versions:
 
 - [Python](https://github.com/pawurb/python-pg-extras)
 
+- [Haskell](https://github.com/pawurb/haskell-pg-extras)
+
 ## Installation
 
 In your Gemfile
@@ -83,6 +85,13 @@ RubyPGExtras.cache_hit(in_format: :raw) =>
  #<PG::Result:0x00007f75777f7328 status=PGRES_TUPLES_OK ntuples=2 nfields=2 cmd_tuples=2>
 ```
 
+Some methods accept an optional `args` param allowing you to customize queries:
+
+```ruby
+RubyPGExtras.long_running_queries(args: { threshold: "200 milliseconds" })
+
+```
+
 ## Available methods
 
 ### `cache_hit`
@@ -114,7 +123,7 @@ RubyPGExtras.index_cache_hit
 (truncated results for brevity)
 ```
 
-The same as `cache_hit` with each table's indexes cache hit info displayed seperately.
+The same as `cache_hit` with each table's indexes cache hit info displayed separately.
 
 ### `table_cache_hit`
 
@@ -167,7 +176,7 @@ RubyPGExtras.locks
 (4 rows)
 ```
 
-This command displays queries that have taken out an exlusive lock on a relation. Exclusive locks typically prevent other operations on that relation from taking place, and can be a cause of "hung" queries that are waiting for a lock to be granted.
+This command displays queries that have taken out an exclusive lock on a relation. Exclusive locks typically prevent other operations on that relation from taking place, and can be a cause of "hung" queries that are waiting for a lock to be granted.
 
 ### `all_locks`
 
@@ -183,7 +192,7 @@ This command displays all the current locks, regardless of their type.
 
 ```ruby
 
-RubyPGExtras.outliers
+RubyPGExtras.outliers(args: { limit: 20 })
 
                    qry                   |    exec_time     | prop_exec_time |   ncalls    | sync_io_time
 -----------------------------------------+------------------+----------------+-------------+--------------
@@ -197,7 +206,7 @@ RubyPGExtras.outliers
 (truncated results for brevity)
 ```
 
-This command displays statements, obtained from `pg_stat_statements`, ordered by the amount of time to execute in aggregate. This includes the statement itself, the total execution time for that statement, the proportion of total execution time for all statements that statement has taken up, the number of times that statement has been called, and the amount of time that statement spent on synchronous I/O (reading/writing from the filesystem).
+This command displays statements, obtained from `pg_stat_statements`, ordered by the amount of time to execute in aggregate. This includes the statement itself, the total execution time for that statement, the proportion of total execution time for all statements that statement has taken up, the number of times that statement has been called, and the amount of time that statement spent on synchronous I/O (reading/writing from the file system).
 
 Typically, an efficient query will have an appropriate ratio of calls to total execution time, with as little time spent on I/O as possible. Queries that have a high total execution time but low call count should be investigated to improve their performance. Queries that have a high proportion of execution time being spent on synchronous I/O should also be investigated.
 
@@ -205,7 +214,7 @@ Typically, an efficient query will have an appropriate ratio of calls to total e
 
 ```ruby
 
-RubyPGExtras.calls
+RubyPGExtras.calls(args: { limit: 10 })
 
                    qry                   |    exec_time     | prop_exec_time |   ncalls    | sync_io_time
 -----------------------------------------+------------------+----------------+-------------+--------------
@@ -306,7 +315,7 @@ RubyPGExtras.table_indexes_size
 (truncated results for brevity)
 ```
 
-This command displays the total size of indexes for each table and materialized view, in MB. It is calcualtes by using the system administration function `pg_indexes_size()`.
+This command displays the total size of indexes for each table and materialized view, in MB. It is calculated by using the system administration function `pg_indexes_size()`.
 
 ### `total_table_size`
 
@@ -330,7 +339,7 @@ This command displays the total size of each table and materialized view in the
 
 ```ruby
 
-RubyPGExtras.unused_indexes
+RubyPGExtras.unused_indexes(args: { min_scans: 20 })
 
           table      |                       index                | index_size | index_scans
 ---------------------+--------------------------------------------+------------+-------------
@@ -342,6 +351,22 @@ RubyPGExtras.unused_indexes
 
 This command displays indexes that have < 50 scans recorded against them, and are greater than 5 pages in size, ordered by size relative to the number of index scans. This command is generally useful for eliminating indexes that are unused, which can impact write performance, as well as read performance should they occupy space in memory.
 
+### `null_indexes`
+
+```ruby
+
+RubyPGExtras.null_indexes
+
+   oid   |         index      | index_size | unique | indexed_column | null_frac | expected_saving
+---------+--------------------+------------+--------+----------------+-----------+-----------------
+  183764 | users_reset_token  | 1418 MB    | t      | reset_token    |   96.15%  | 1363 MB
+   88732 | plan_cancelled_at  | 1651 MB    | f      | cancelled_at   |    6.11%  | 101 MB
+ 9827345 | users_email        | 22 MB      | t      | email          |   11.21%  | 2494 kB
+
+```
+
+This commands displays indexes that contain `NULL` values. A high ratio of `NULL` values means that using a partial index excluding them will be beneficial in case they are not used for searching. [Source and more info](https://hakibenita.com/postgresql-unused-index-size).
+
 ### `seq_scans`
 
 ```ruby
@@ -362,7 +387,7 @@ RubyPGExtras.seq_scans
 (truncated results for brevity)
 ```
 
-This command displays the number of sequential scans recorded against all tables, descending by count of sequential scans. Tables that have very high numbers of sequential scans may be underindexed, and it may be worth investigating queries that read from these tables.
+This command displays the number of sequential scans recorded against all tables, descending by count of sequential scans. Tables that have very high numbers of sequential scans may be under-indexed, and it may be worth investigating queries that read from these tables.
 
 ### `long_running_queries`
 
@@ -435,7 +460,7 @@ RubyPGExtras.vacuum_stats
  (truncated results for brevity)
 ```
 
-This command displays statistics related to vacuum operations for each table, including an estiamtion of dead rows, last autovacuum and the current autovacuum threshold. This command can be useful when determining if current vacuum thresholds require adjustments, and to determine when the table was last vacuumed.
+This command displays statistics related to vacuum operations for each table, including an estimation of dead rows, last autovacuum and the current autovacuum threshold. This command can be useful when determining if current vacuum thresholds require adjustments, and to determine when the table was last vacuumed.
 
 ### `kill_all`
 

data/lib/ruby-pg-extras.rb

@@ -10,26 +10,37 @@ module RubyPGExtras
   QUERIES = %i(
     bloat blocking cache_hit
     calls extensions table_cache_hit index_cache_hit
-    index_size index_usage locks all_locks
+    index_size index_usage null_indexes locks all_locks
     long_running_queries mandelbrot outliers
     records_rank seq_scans table_indexes_size
     table_size total_index_size total_table_size
     unused_indexes vacuum_stats kill_all
   )
 
+  DEFAULT_ARGS = Hash.new({}).merge({
+    calls: { limit: 10 },
+    long_running_queries: { threshold: "500 milliseconds" },
+    outliers: { limit: 10 },
+    unused_indexes: { min_scans: 50 }
+  })
+
   QUERIES.each do |query_name|
-    define_singleton_method query_name do |options = { in_format: :display_table }|
+    define_singleton_method query_name do |options = {}|
       run_query(
         query_name: query_name,
-        in_format: options.fetch(:in_format)
+        in_format: options.fetch(:in_format, :display_table),
+        args: options.fetch(:args, {})
       )
     end
   end
 
-  def self.run_query(query_name:, in_format:)
-    result = connection.exec(
+  def self.run_query(query_name:, in_format:, args: {})
+    sql = if (custom_args = DEFAULT_ARGS[query_name].merge(args)) != {}
+      sql_for(query_name: query_name) % custom_args
+    else
       sql_for(query_name: query_name)
-    )
+    end
+    result = connection.exec(sql)
 
     display_result(
       result,

data/lib/ruby-pg-extras/queries/calls.sql

@@ -1,9 +1,9 @@
-/* 10 queries that have highest frequency of execution */
+/* Queries that have highest frequency of execution */
 
 SELECT query AS qry,
 interval '1 millisecond' * total_time AS exec_time,
-to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%'  AS prop_exec_time,
+to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
 to_char(calls, 'FM999G999G990') AS ncalls,
 interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time
 FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
-ORDER BY calls DESC LIMIT 10;
+ORDER BY calls DESC LIMIT %{limit};

data/lib/ruby-pg-extras/queries/long_running_queries.sql

@@ -1,4 +1,4 @@
-/* All queries longer than five minutes by descending duration */
+/* All queries longer than the threshold by descending duration */
 
 SELECT
   pid,
@@ -9,6 +9,6 @@ FROM
 WHERE
   pg_stat_activity.query <> ''::text
   AND state <> 'idle'
-  AND now() - pg_stat_activity.query_start > interval '5 minutes'
+  AND now() - pg_stat_activity.query_start > interval '%{threshold}'
 ORDER BY
   now() - pg_stat_activity.query_start DESC;

data/lib/ruby-pg-extras/queries/null_indexes.sql

@@ -0,0 +1,32 @@
+/* Find indexed columns with high null_frac */
+SELECT
+    c.oid,
+    c.relname AS index,
+    pg_size_pretty(pg_relation_size(c.oid)) AS index_size,
+    i.indisunique AS unique,
+    a.attname AS indexed_column,
+    CASE s.null_frac
+        WHEN 0 THEN ''
+        ELSE to_char(s.null_frac * 100, '999.00%')
+    END AS null_frac,
+    pg_size_pretty((pg_relation_size(c.oid) * s.null_frac)::bigint) AS expected_saving
+FROM
+    pg_class c
+    JOIN pg_index i ON i.indexrelid = c.oid
+    JOIN pg_attribute a ON a.attrelid = c.oid
+    JOIN pg_class c_table ON c_table.oid = i.indrelid
+    JOIN pg_indexes ixs ON c.relname = ixs.indexname
+    LEFT JOIN pg_stats s ON s.tablename = c_table.relname AND a.attname = s.attname
+WHERE
+    -- Primary key cannot be partial
+    NOT i.indisprimary
+    -- Exclude already partial indexes
+    AND i.indpred IS NULL
+    -- Exclude composite indexes
+    AND array_length(i.indkey, 1) = 1
+    -- Exclude indexes without null_frac ratio
+    AND coalesce(s.null_frac, 0) != 0
+    -- Larger than 10MB
+    AND pg_relation_size(c.oid) > 10 * 1024 ^ 2
+ORDER BY
+  pg_relation_size(c.oid) * s.null_frac DESC;

data/lib/ruby-pg-extras/queries/outliers.sql

@@ -1,10 +1,10 @@
-/* 10 queries that have longest execution time in aggregate */
+/* Queries that have longest execution time in aggregate */
 
 SELECT interval '1 millisecond' * total_time AS total_exec_time,
-to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%'  AS prop_exec_time,
+to_char((total_time/sum(total_time) OVER()) * 100, 'FM90D0') || '%%'  AS prop_exec_time,
 to_char(calls, 'FM999G999G999G990') AS ncalls,
 interval '1 millisecond' * (blk_read_time + blk_write_time) AS sync_io_time,
 query AS query
 FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1)
 ORDER BY total_time DESC
-LIMIT 10;
+LIMIT %{limit};

data/lib/ruby-pg-extras/queries/unused_indexes.sql

@@ -11,6 +11,6 @@ SELECT
   idx_scan as index_scans
 FROM pg_stat_user_indexes ui
 JOIN pg_index i ON ui.indexrelid = i.indexrelid
-WHERE NOT indisunique AND idx_scan < 50 AND pg_relation_size(relid) > 5 * 8192
+WHERE NOT indisunique AND idx_scan < %{min_scans} AND pg_relation_size(relid) > 5 * 8192
 ORDER BY pg_relation_size(i.indexrelid) / nullif(idx_scan, 0) DESC NULLS FIRST,
 pg_relation_size(i.indexrelid) DESC;

data/lib/ruby-pg-extras/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubyPGExtras
-  VERSION = "1.2.3"
+  VERSION = "1.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-pg-extras
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.5.0
 platform: ruby
 authors:
 - pawurb
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-08 00:00:00.000000000 Z
+date: 2021-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -96,6 +96,7 @@ files:
 - lib/ruby-pg-extras/queries/locks.sql
 - lib/ruby-pg-extras/queries/long_running_queries.sql
 - lib/ruby-pg-extras/queries/mandelbrot.sql
+- lib/ruby-pg-extras/queries/null_indexes.sql
 - lib/ruby-pg-extras/queries/outliers.sql
 - lib/ruby-pg-extras/queries/records_rank.sql
 - lib/ruby-pg-extras/queries/seq_scans.sql
@@ -129,7 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Ruby PostgreSQL performance database insights