pg_metrics 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f02f742e23e3d02686a15c2cf6a9f8b48b527e3
-  data.tar.gz: cb0e25e46b852b7cc3f3551b3c87d63bbf187acb
+  metadata.gz: baa62b576696fc88002fa07901d434d41ce5bfdf
+  data.tar.gz: e42fe1cfb12b58e4d20f63cf7408062c23bd589a
 SHA512:
-  metadata.gz: 615b4a325678b94b2f3596f924c8a5ac978314d2a4b0a18d09eb6785d45882545291948298b2ba849f20efd4b4e075cda241d7b091584a5ac1f60c656296f7df
-  data.tar.gz: 455211aeb1ef3fa4883edbee01bc70d9111433dfe9e04950134765097a8f7664d131b31e1f6b26e3e24a46e3422e6146a85ffc482bdf01c2481de26a8b9dc236
+  metadata.gz: 8d74d6b1239d90b66401e25862401d68ec3fe8ecdd7cc5de670eaea3485ccbae5fb10012b9f7d7cc6997df6ec3ff03f5d6adea439070f37e1cec30e8416b1e20
+  data.tar.gz: ca5d76a312f00266829cf8e6193093417769e17296690c5660b84f45a60ac8449700409ee456a8a020ca731f3433d8bba2132c902ed74581d1507f825ccd4d92

data/CHANGELOG.markdown

@@ -1,5 +1,19 @@
 # pg_metrics Changelog
 
+## Changes between 0.1.1 and 0.2.0
+
+### Add --table-free-space and --index-ideal-size options
+
+With `--table-free-space`, pg_metrics collects free space stats
+(provided `pg_freespacemap` is installed). With `--index-ideal-sizes`,
+`pg_metrics` estimates how large an index *should* be, assuming it has
+no dead tuples. Both of these options are not included by default.
+
+Motivated by the fact that `pg_freespace` in postgres versions 8.4
+and up is quite slow and we don't want to run it as frequently as
+other metrics, add an `--only` option that runs only the options
+explicitly specified.
+
 ## Changes between 0.1.1 and 0.0.7
 
 ### Remove sensu

data/README.markdown

@@ -20,7 +20,7 @@ To collect PostgreSQL database metrics for the `prod` database, include the
 
     pg_metrics_statsd --host localhost --port 8125 --connection "host=localhost port=5432" --dbname=prod
 
-By default, pg_metrics_statsd collects stats from  `pg_locks`,
+By default, `pg_metrics_statsd` collects stats from  `pg_locks`,
 `pg_stat_user_functions` (where available), `pg_stat_user_tables`,
 `pg_stat_user_tables`, `pg_stat_user_indexes`, `pg_statio_user_indexes`,
 as well as per-table and per-index sizes. You can omit stats by supplying
@@ -35,6 +35,26 @@ command line flags:
  - `--no-table-sizes`
  - `--no-index-sizes`
 
+You can also use the `--only` flag to collect *only* the explicitly specified
+stats. For example, to collect only table and index size stats:
+
+    pg_metrics_statsd --only --table-sizes --index-sizes --dbname=prod
+
+### Free space
+`pg_metrics_statsd` can also collect table free space metrics provided the
+`pg_freespacemap` contrib module is installed. To include free space metrics,
+pass the `--table-free-space` flag along with the database. Collecting free
+space metrics is not included by default.
+
+For postgres versions 8.4 and higher, free space metrics take a while to generate.
+
+### Index ideal sizes
+`pg_metrics_statsd` includes an option to measure *ideal index size*, which is
+an estimate of how much disk space an index *should* require if it contains no
+dead tuples. This can be useful, along with index sizes, to estimate how bloated
+indexes are. To collect ideal index size metrics, pass the `--index-ideal-sizes`
+flag.
+
 ### pgbouncer metrics
 
 `pg_metrics` can also collect `pgbouncer` metrics by passing the `--pgbouncer`

data/lib/pg_metrics/metrics.rb

@@ -12,6 +12,8 @@ module PgMetrics
     TableStats = :table_stats
     IndexStatio = :index_statio
     IndexStats = :index_stats
+    TableFreeSpace = :table_free_space
+    IndexIdealSizes = :index_ideal_size
 
     def self.fetch_instance_metrics(app_name, conn_info, regexp = nil)
       metrics = []
@@ -231,7 +233,7 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
                       calls, total_time, self_time
           FROM pg_stat_user_functions
           JOIN pg_proc ON pg_proc.oid = funcid
-          WHERE schemaname NOT IN ('information_schema', 'pg_catalog')) AS funcs}
+          WHERE schemaname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')) AS funcs}
           : nil
         },
 
@@ -262,7 +264,29 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
        FROM pg_class r
        JOIN pg_namespace n ON r.relnamespace = n.oid
        WHERE r.relkind = 'r'
-         AND n.nspname NOT IN ('pg_catalog', 'information_schema')}
+         AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
+        },
+
+        TableFreeSpace => {
+          prefix: %w(table),
+          query: Gem::Version.new(server_version) >= Gem::Version.new('8.4') \
+          ? %{SELECT n.nspname AS key, t.relname AS key2,
+                     COALESCE((SELECT sum(pg_freespace.avail) AS sum
+                                 FROM pg_freespace(t.oid::regclass) AS pg_freespace(blkno, avail)), 0::bigint) AS free_space
+                FROM pg_class t
+                JOIN pg_namespace n ON t.relnamespace = n.oid
+                WHERE t.relkind = 'r'::"char"
+                      AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
+          : %{SELECT n.nspname AS key, t.relname AS key2, fsm.bytes AS free_space
+                FROM pg_class t
+                JOIN pg_namespace n ON t.relnamespace = n.oid
+                LEFT JOIN (SELECT fsm.relfilenode, sum(fsm.bytes) AS bytes
+                             FROM pg_freespacemap_pages fsm
+                             JOIN pg_database db ON db.oid = fsm.reldatabase
+                                                    AND db.datname = current_database()
+                             GROUP BY fsm.relfilenode) fsm ON t.relfilenode = fsm.relfilenode
+                WHERE t.relkind = 'r'::"char"
+                      AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
         },
 
         IndexSizes => {
@@ -273,7 +297,60 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
        JOIN pg_class cr ON cr.oid = i.indrelid
        JOIN pg_namespace n on ci.relnamespace = n.oid
        WHERE ci.relkind = 'i' AND cr.relkind = 'r'
-             AND n.nspname NOT IN ('pg_catalog', 'information_schema')}
+             AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
+        },
+
+        IndexIdealSizes => {
+          prefix: %w(table),
+          query: %{SELECT pg_namespace.nspname AS key, rel.relname AS key2,
+       'index' AS key3, idx.relname AS key4,
+       ((ceil(idx.reltuples * ((constants.index_tuple_header_size
+                                  + constants.item_id_data_size
+                                  + CASE WHEN (COALESCE(sum(CASE WHEN statts.staattnotnull THEN 0 ELSE 1 END), 0::bigint)
+                                               + ((SELECT COALESCE(sum(CASE WHEN atts.attnotnull THEN 0 ELSE 1 END), 0::bigint) AS "coalesce"
+                                                     FROM pg_attribute atts
+                                                     JOIN (SELECT pg_index.indkey[the.i] AS attnum
+                                                             FROM generate_series(0, pg_index.indnatts - 1) the(i)) cols ON atts.attnum = cols.attnum
+                                                             WHERE atts.attrelid = pg_index.indrelid))) > 0
+                                         THEN (SELECT the.null_bitmap_size
+                                                        + constants.max_align
+                                                        - CASE WHEN (the.null_bitmap_size % constants.max_align) = 0
+                                                                 THEN constants.max_align
+                                                               ELSE the.null_bitmap_size % constants.max_align END
+                                                 FROM (VALUES (ceil(pg_index.indnatts::real / 8)::int)) the (null_bitmap_size))
+                                         ELSE 0 END)::double precision
+                                 + COALESCE(sum(statts.stawidth::double precision * (1::double precision - statts.stanullfrac)), 0::double precision)
+                                 + COALESCE((SELECT sum(atts.stawidth::double precision * (1::double precision - atts.stanullfrac)) AS sum
+                                               FROM pg_statistic atts
+                                               JOIN (SELECT pg_index.indkey[the.i] AS attnum
+                                                       FROM generate_series(0, pg_index.indnatts - 1) the(i)) cols ON atts.staattnum = cols.attnum
+                                               WHERE atts.starelid = pg_index.indrelid), 0::double precision))
+                            / (constants.block_size - constants.page_header_data_size::numeric - constants.special_space::numeric)::double precision)
+           + constants.index_metadata_pages::double precision)
+          * constants.block_size::double precision)::bigint AS ideal_size
+  FROM pg_index
+  JOIN pg_class idx ON pg_index.indexrelid = idx.oid
+  JOIN pg_class rel ON pg_index.indrelid = rel.oid
+  JOIN pg_namespace ON idx.relnamespace = pg_namespace.oid
+  LEFT JOIN (SELECT pg_statistic.starelid, pg_statistic.staattnum, pg_statistic.stanullfrac, pg_statistic.stawidth, pg_attribute.attnotnull AS staattnotnull
+               FROM pg_statistic
+               JOIN pg_attribute ON (pg_statistic.starelid,pg_statistic.staattnum) = (pg_attribute.attrelid, pg_attribute.attnum)) statts
+    ON statts.starelid = idx.oid
+  CROSS JOIN (SELECT current_setting('block_size'::text)::numeric AS block_size,
+                      CASE WHEN "substring"(version(), 12, 3) = ANY (ARRAY['8.0'::text, '8.1'::text, '8.2'::text]) THEN 27
+                           ELSE 23 END AS tuple_header_size,
+                      CASE WHEN version() ~ 'mingw32'::text THEN 8 ELSE 4 END AS max_align,
+                      8 AS index_tuple_header_size,
+                      4 AS item_id_data_size,
+                      24 AS page_header_data_size,
+                      0 AS special_space,
+                      1 AS index_metadata_pages) AS constants
+  WHERE nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+  GROUP BY pg_namespace.nspname, rel.relname, rel.oid, idx.relname, idx.reltuples, idx.relpages, pg_index.indexrelid,
+           pg_index.indrelid, pg_index.indkey, pg_index.indnatts,
+           constants.block_size, constants.tuple_header_size, constants.max_align,
+           constants.index_tuple_header_size, constants.item_id_data_size, constants.page_header_data_size,
+           constants.index_metadata_pages, constants.special_space;}
         },
 
         TableStatio => {

data/lib/pg_metrics/statsd.rb

@@ -44,21 +44,24 @@ module PgMetrics
     end
 
     def self.parse(args)
+      default_stats = [PgMetrics::Metrics::Functions,
+                       PgMetrics::Metrics::Locks,
+                       PgMetrics::Metrics::TableSizes,
+                       PgMetrics::Metrics::IndexSizes,
+                       PgMetrics::Metrics::TableStatio,
+                       PgMetrics::Metrics::TableStats,
+                       PgMetrics::Metrics::IndexStatio,
+                       PgMetrics::Metrics::IndexStats].to_set
       options = {
         host: "localhost",
         port: 8125,
         conn: "",
         scheme: %(#{Socket.gethostname}.postgresql),
-        dbstats: [PgMetrics::Metrics::Functions,
-                  PgMetrics::Metrics::Locks,
-                  PgMetrics::Metrics::TableSizes,
-                  PgMetrics::Metrics::IndexSizes,
-                  PgMetrics::Metrics::TableStatio,
-                  PgMetrics::Metrics::TableStats,
-                  PgMetrics::Metrics::IndexStatio,
-                  PgMetrics::Metrics::IndexStats].to_set
+        dbstats: Set.new
       }
 
+      stats = { added: Set.new, default: default_stats }
+
       OptionParser.new do |opts|
         opts.on("-h", "--host STATSD_HOST", "StatsD host") { |v| options[:host] = v }
         opts.on("-p", "--port STATSD_PORT", "StatsD port") { |v| options[:port] = v.to_i }
@@ -66,20 +69,34 @@ module PgMetrics
         opts.on("-d", "--dbname DBNAME", "PostgreSQL database name for database metrics") { |v| options[:dbname] = v }
         opts.on("-e", "--exclude REGEXP", "Exclude objects matching given regexp") { |v| options[:exclude] = ::Regexp.new(v) }
         opts.on("-s", "--scheme SCHEME", "Metric namespace") { |v| options[:scheme] = v }
-        opts.on("--[no-]functions", "Collect database function stats") { |v| options[:dbstats].delete(PgMetrics::Metrics::Functions) unless v }
-        opts.on("--[no-]locks", "Collect database lock stats") { |v| options[:dbstats].delete(PgMetrics::Metrics::Locks) unless v }
-        opts.on("--[no-]table-sizes", "Collect database table size stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableSizes) unless v }
-        opts.on("--[no-]index-sizes", "Collect database index size stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexSizes) unless v }
-        opts.on("--[no-]table-statio", "Collect database table statio stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableStatio) unless v }
-        opts.on("--[no-]table-stats", "Collect database table stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableStats) unless v }
-        opts.on("--[no-]index-statio", "Collect database index statio stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexStatio) unless v }
-        opts.on("--[no-]index-stats", "Collect database index stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexStats) unless v }
+        opts.on("--only", "Collect only specified stats") { |v| stats[:default] = Set.new }
+        opts.on("--[no-]functions", "Collect database function stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::Functions, v) }
+        opts.on("--[no-]locks", "Collect database lock stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::Locks, v) }
+        opts.on("--[no-]table-sizes", "Collect database table size stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableSizes, v) }
+        opts.on("--[no-]index-sizes", "Collect database index size stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexSizes, v) }
+        opts.on("--[no-]table-statio", "Collect database table statio stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableStatio, v) }
+        opts.on("--[no-]table-stats", "Collect database table stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableStats, v) }
+        opts.on("--[no-]index-statio", "Collect database index statio stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexStatio, v) }
+        opts.on("--[no-]index-stats", "Collect database index stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexStats, v) }
+        opts.on("--[no-]table-free-space", "Collect database table free space stats (requires pg_freespacemap)") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableFreeSpace, v) }
+        opts.on("--[no-]index-ideal-sizes", "Collect database index ideal size estimates") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexIdealSizes, v) }
         opts.on("--pgbouncer", "Collect pgbouncer stats") { |v| options[:pgbouncer] = true }
         opts.on("--verbose") { |v| options[:verbose] = true }
         opts.on("--version") { |v| options[:version] = v }
       end.order!(args)
 
+      options[:dbstats] = stats[:added].merge(stats[:default])
       options
     end
+
+    def self.mutate_stats(stats, key, do_add)
+      if do_add
+        stats[:added].add(key)
+      else
+        stats[:default].delete(key)
+      end
+      stats
+    end
+
   end
 end

data/lib/pg_metrics/version.rb

@@ -1,3 +1,3 @@
 module PgMetrics
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/pg_metrics.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.name = "pg_metrics"
   spec.version = PgMetrics::VERSION
   spec.licenses = %w(MIT)
-  spec.date = "2015-03-17"
+  spec.date = "2015-03-19"
   spec.summary = "pg_metrics"
   spec.description = "PostgreSQL Metrics"
   spec.authors = ["Michael Glaesemann"]

data/test/test_statsd.rb

@@ -53,7 +53,7 @@ module PgMetrics
                     PgMetrics::Metrics::TableStats,
                     PgMetrics::Metrics::IndexStatio,
                     PgMetrics::Metrics::IndexStats].to_set
-        assert_equal(config[:dbstats], expected)
+        assert_equal(expected, config[:dbstats])
       end
 
       def test_should_set_all_metrics_with_positive_locks
@@ -67,7 +67,7 @@ module PgMetrics
                     PgMetrics::Metrics::TableStats,
                     PgMetrics::Metrics::IndexStatio,
                     PgMetrics::Metrics::IndexStats].to_set
-        assert_equal(config[:dbstats], expected)
+        assert_equal(expected, config[:dbstats])
       end
 
       def test_should_not_collect_locks
@@ -80,14 +80,36 @@ module PgMetrics
                     PgMetrics::Metrics::TableStats,
                     PgMetrics::Metrics::IndexStatio,
                     PgMetrics::Metrics::IndexStats].to_set
-        assert_equal(config[:dbstats], expected)
+        assert_equal(expected, config[:dbstats])
       end
 
       def test_should_remove_all_but_locks
         args = %w(--no-functions --no-table-sizes --no-index-sizes --no-table-statio --no-table-stats --no-index-stats --no-index-statio)
         config = PgMetrics::Statsd::parse(args)
         expected = [PgMetrics::Metrics::Locks].to_set
-        assert_equal(config[:dbstats], expected)
+        assert_equal(expected, config[:dbstats])
+      end
+
+      def test_should_removal_all_but_locks_using_only
+        args = %w(--only --locks)
+        config = PgMetrics::Statsd::parse(args)
+        expected = [PgMetrics::Metrics::Locks].to_set
+        assert_equal(expected, config[:dbstats])
+      end
+
+      def test_should_only_include_table_free_space
+        args = %w(--only --table-free-space)
+        config = PgMetrics::Statsd::parse(args)
+        expected = [PgMetrics::Metrics::TableFreeSpace].to_set
+        assert_equal(expected, config[:dbstats])
+      end
+
+      def test_should_only_include_table_free_space
+        args = %w(--only --index-ideal-sizes --table-free-space)
+        config = PgMetrics::Statsd::parse(args)
+        expected = [PgMetrics::Metrics::TableFreeSpace,
+                    PgMetrics::Metrics::IndexIdealSizes].to_set
+        assert_equal(expected, config[:dbstats])
       end
 
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_metrics
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Glaesemann
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-17 00:00:00.000000000 Z
+date: 2015-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg