pg_metrics 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1f02f742e23e3d02686a15c2cf6a9f8b48b527e3
+  data.tar.gz: cb0e25e46b852b7cc3f3551b3c87d63bbf187acb
+SHA512:
+  metadata.gz: 615b4a325678b94b2f3596f924c8a5ac978314d2a4b0a18d09eb6785d45882545291948298b2ba849f20efd4b4e075cda241d7b091584a5ac1f60c656296f7df
+  data.tar.gz: 455211aeb1ef3fa4883edbee01bc70d9111433dfe9e04950134765097a8f7664d131b31e1f6b26e3e24a46e3422e6146a85ffc482bdf01c2481de26a8b9dc236

data/CHANGELOG.markdown

@@ -0,0 +1,90 @@
+# pg_metrics Changelog
+
+## Changes between 0.1.1 and 0.0.7
+
+### Remove sensu
+
+pg_metrics generates too many stats for sensu to cope with, which
+is why we made the statsd version in the first place. As we're not
+using or testing the sensu version, remove it rather than let it bitrot.
+
+### Update project homepage
+
+Make the project public (yay!), making github.com the official homepage.
+
+## Changes between 0.0.7 and 0.0.6
+
+### Use client connection user in backend stats
+
+In 0.0.5, backend-stats for automatic connections or non-force-user database
+definitions used `SAMEUSER` as the username for the backend connection. The
+actual client connection user is now used instead.
+
+### Drop backend stats for pools with no corresponding backend
+
+`pgbouncer` sometimes shows databases in `SHOW pools` that have no
+corresponding entry in `SHOW databases`. These pool entries are
+now ignored for backend stats.
+
+## Changes between 0.0.5 and 0.0.6
+
+### Add pgbouncer stats
+
+`pg_metrics` can now be used to collect pgbouncer stats using the `--pgbouncer`
+flag. Only results of `SHOW stats` and `SHOW pools` are collected, as well
+as per-backend stats, which uses the results of `SHOW databases` to aggregate
+pool stats per backend-connection, not just per defined database connection.
+
+## Changes between 0.0.4 and 0.0.5
+
+### Collect waiting session stats
+
+Count sessions as *waiting* if `pg_stat_activity.waiting` is `TRUE`,
+and as the session state otherwise (using `pg_stat_activity.state` for
+PostgreSQL versions >= 9.2 and calculated from `pg_stat_activity.current_query`
+in earlier versions).
+
+`pg_stat_activity` does track waiting idependently of state, but for practical
+purposes not all that useful to track them separately for metrics collection.
+
+### Track xlog.location for PostgreSQL versions <= 9.0
+
+Earlier versions of pg_metrics did not collect `current_xlog_location`
+for versions earlier than 9.1. The `pg_current_xlog_location()` function
+is available in both PostgreSQL 8.3 and 8.4. The `pg_last_xlog_(receive|replay)_location()`
+functions are available for PostgreSQL versions >= 9.0, so collect those
+when available.
+
+
+## Changes between 0.0.3 and 0.0.4
+
+### Allow specfication of which database stats are collected.
+
+Prior to 0.0.4, per-database stats were collected only from `pg_locks`,
+`pg_stat_user_tables` and `pg_statio_user_tables`. `pg_metrics_statsd`
+collects all stats by default, and allows specification of which stats
+to omit with a a variety of `--no-*` command line flags.
+
+## Changes between 0.0.2 and 0.0.3
+
+### Improve formatting of verbose output
+
+0.0.3 prints each metric on its own line.
+
+### Permit using short -s flag to specify scheme
+
+0.0.3 allows you to specify scheme using `-s SCHEME` as well
+as the legacy `--scheme SCHEME`.
+
+## Changes between 0.0.1 and 0.0.2
+
+### Fix use of regexp filter
+
+A incomplete refactor left behind a second instantiation of the filter regex,
+along with a reference to a variable that was no longer in scope.
+
+## Set application_name only for PostgreSQL versions >= 9.0
+
+The application_name parameter was introduced in PostgreSQL version 9.0. Earlier
+versions (such as 8.3 and 8.4) will throw an error if you try to set it, so we
+no longer try to set it for versions that don't support it.

data/DEV.markdown

@@ -0,0 +1,22 @@
+# Development notes
+
+Processes I often forget between release cycles
+
+## To build and install locally
+
+    gem build pg_metrics.gemspec
+    gem install ./pg_metrics-0.0.X.gem
+
+
+## To release
+
+* Update `CHANGELOG.markdown`
+* Update `lib/pg_metrics/version.rb`
+* Update `spec.date` in `pg_metrics.gemspec`
+* Tag
+
+        git tag -a "v0.0.X" -m "version 0.0.X"
+
+* Push to github repo
+
+        git push origin --tags

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+gem "pg"
+gem "statsd-ruby"
+gem "simplecov", :require => false, :group => :test

data/Gemfile.lock

@@ -0,0 +1,20 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    docile (1.1.5)
+    multi_json (1.10.1)
+    pg (0.17.1)
+    simplecov (0.9.1)
+      docile (~> 1.1.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    statsd-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pg
+  simplecov
+  statsd-ruby

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2015, MeetMe, Inc.
+All rights reserved.
+
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,43 @@
+# pg_metrics PostgreSQL Metrics
+
+`pg_metrics` is a PostgreSQL metrics collector for use with statsd.
+
+## Installation
+
+    gem install pg_metrics
+
+The `pg_metrics_statsd` command is now available.
+
+## Usage
+
+To collect PostgreSQL instance metrics on localhost port 5432 and pass them to a
+statsd instance running on localhost port 8125:
+
+    pg_metrics_statsd --host localhost --port 8125 --connection "host=localhost port=5432"
+
+To collect PostgreSQL database metrics for the `prod` database, include the
+`--dbname` parameter:
+
+    pg_metrics_statsd --host localhost --port 8125 --connection "host=localhost port=5432" --dbname=prod
+
+By default, pg_metrics_statsd collects stats from  `pg_locks`,
+`pg_stat_user_functions` (where available), `pg_stat_user_tables`,
+`pg_stat_user_tables`, `pg_stat_user_indexes`, `pg_statio_user_indexes`,
+as well as per-table and per-index sizes. You can omit stats by supplying
+command line flags:
+
+ - `--no-functions`
+ - `--no-locks`
+ - `--no-table-stats`
+ - `--no-table-statio`
+ - `--no-index-stats`
+ - `--no-index-statio`
+ - `--no-table-sizes`
+ - `--no-index-sizes`
+
+### pgbouncer metrics
+
+`pg_metrics` can also collect `pgbouncer` metrics by passing the `--pgbouncer`
+flag.
+
+    pg_metrics_statsd --host localhost --port 8125 --connection "host=localhost port=6432" --pgbouncer

data/bin/pg_metrics_statsd

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+begin
+  require 'pg_metrics'
+rescue LoadError
+  require 'rubygems'
+  require 'pg_metrics'
+end
+
+begin
+  exit PgMetrics::Statsd::main(ARGV)
+rescue => e
+  STDERR.puts e.message
+  STDERR.puts e.backtrace
+end

data/lib/pg_metrics/metrics.rb

@@ -0,0 +1,347 @@
+require "pg"
+require "set"
+
+module PgMetrics
+  module Metrics
+
+    Functions = :functions
+    Locks = :locks
+    TableSizes = :table_size
+    IndexSizes = :index_size
+    TableStatio = :table_statio
+    TableStats = :table_stats
+    IndexStatio = :index_statio
+    IndexStats = :index_stats
+
+    def self.fetch_instance_metrics(app_name, conn_info, regexp = nil)
+      metrics = []
+      conn = make_conn(conn_str(conn_info), app_name)
+      server_version = conn.parameter_status("server_version")
+      instance_metrics(server_version).values.each do |m|
+        metrics += fetch_metrics(conn, m[:prefix], m[:query])
+      end
+      conn.finish
+      filter_metrics(metrics, regexp)
+    end
+
+    def self.fetch_database_metrics(app_name, conn_info, dbname, select_names, regexp = nil)
+      metrics = []
+      conn = make_conn(conn_str(conn_info, dbname), app_name)
+      server_version = conn.parameter_status("server_version")
+      select_metrics = database_metrics(server_version).select { |k, _v| select_names.include? k }
+      select_metrics.values.each do |m|
+        metrics += fetch_metrics(conn, ["database", dbname] + m[:prefix], m[:query])
+      end
+      conn.finish
+      filter_metrics(metrics, regexp)
+    end
+
+    def self.make_conn(conn_str, app_name)
+      conn = PG::Connection.new(conn_str)
+      server_version = conn.parameter_status("server_version")
+      conn.exec(%(SET application_name = "#{app_name}")) if Gem::Version.new(server_version) >= Gem::Version.new("9.0")
+      conn
+    end
+
+    def self.filter_metrics(metrics, regexp = nil)
+      metrics.reject! { |m| m[1].nil? }
+      metrics.reject! { |m| m[0].any? { |k| k =~ regexp  } } if regexp
+      metrics
+    end
+
+    def self.conn_str(conn_info, dbname = "postgres")
+      [conn_info, %(dbname=#{dbname})].join(" ")
+    end
+
+    def self.fetch_metrics(conn, keys, query)
+      metrics = []
+
+      return metrics if query.nil?
+
+      timestamp = Time.now.to_i
+
+      conn.exec(query) do |result|
+        if result.nfields == 1 && result.ntuples == 1
+          # Typically result of SHOW command
+          metrics << format_metric(keys, result.getvalue(0, 0), timestamp)
+        elsif result.nfields >= 2 && result.fields.first == "key"
+          if result.fields.last == "value"
+            # Omit "value" column from metric name
+            nkeys = result.nfields - 1
+            result.each_row do |row|
+              mkeys = row.first(nkeys)
+              value = row.last
+              metrics << format_metric(keys + mkeys, value, timestamp)
+            end
+          else
+            # Use any column named key* as part of the metric name.
+            # Any other columns are named values.
+            nkeys = result.fields.take_while { |f| f =~ /^key/ }.count
+            keycols = result.fields.first(nkeys)
+            nvals = result.nfields - nkeys
+            valcols = result.fields.last(nvals)
+            result.each do |tup|
+              mkeys = keycols.map { |col| tup[col] }
+              valcols.each do |key|
+                value = tup[key]
+                metrics << format_metric(keys + mkeys + [key], value, timestamp)
+              end
+            end
+          end
+        else # We've got a single-row result where columns are named values
+          result[0].each do |key, value|
+            metrics << format_metric(keys + [key], value, timestamp)
+          end
+        end
+      end
+
+      metrics
+    end
+
+    def self.format_metric(keys, value, timestamp)
+      segs = keys.reject { |k| k.nil? }.map { |x| x.gsub(/[\s.]/, "_") }
+      value = decode_xlog_location(value)
+      [segs, value, timestamp]
+    end
+
+    def self.decode_xlog_location(val)
+      return val if val.nil?
+      if (m = val.match(%r{([A-Fa-f0-9]+)/([A-Fa-f0-9]+)}))
+        return (m[1].hex << 32) + m[2].hex
+      end
+      val
+    end
+
+    def self.instance_metrics(server_version)
+      {
+        max_connections: {
+          prefix: %w(config instance max_connections),
+          query: %q{SHOW max_connections}
+        },
+
+        superuser_connections: {
+          prefix: %w(config instance superuser_reserved_connections),
+          query: %q{SHOW superuser_reserved_connections}
+        },
+
+        archive_files: {
+          prefix: %w(archive_files),
+          query: %q{SELECT CAST(COALESCE(SUM(CAST(archive_file ~ E'\\.ready$' AS int)), 0) AS INT) AS ready,
+                         CAST(COALESCE(SUM(CAST(archive_file ~ E'\\.done$' AS int)), 0) AS INT) AS done
+                    FROM pg_catalog.pg_ls_dir('pg_xlog/archive_status') AS archive_files (archive_file)}
+        },
+
+        bgwriter: {
+          prefix: %w(bgwriter),
+          query: %q{SELECT checkpoints_timed, checkpoints_req, buffers_checkpoint,
+              buffers_clean, maxwritten_clean, buffers_backend, buffers_alloc
+       FROM pg_stat_bgwriter}
+        },
+
+        sessions: {
+          prefix: %w(sessions),
+          query: Gem::Version.new(server_version) >= Gem::Version.new('9.2') \
+          ? %{SELECT datname AS key, usename AS key2,
+                     CASE WHEN waiting THEN 'waiting' ELSE state END AS key3,
+                     count(*) AS value
+              FROM pg_stat_activity
+              WHERE pid <> pg_backend_pid() GROUP BY datname, usename, 3}
+          : %{SELECT datname AS key, usename AS key2,
+                CASE WHEN waiting THEN 'waiting'
+                     ELSE CASE current_query
+                               WHEN NULL THEN 'disabled'
+                               WHEN '<IDLE>' THEN 'idle'
+                               WHEN '<IDLE> in transaction' THEN 'idle in transaction'
+                               ELSE 'active' END END AS key3,
+                count(*) AS value
+            FROM pg_stat_activity
+            WHERE procpid <> pg_backend_pid() GROUP BY datname, usename, 3}
+        },
+
+        database_connection_limits: {
+          prefix: %w(config database),
+          query: %q{SELECT datname AS key,
+              CASE WHEN datconnlimit <> -1 THEN datconnlimit ELSE current_setting('max_connections')::int END AS connection_limit
+       FROM pg_database
+       WHERE datallowconn AND NOT datistemplate}
+        },
+
+        user_connection_limits: {
+          prefix: %w(config user),
+          query: %q{SELECT rolname AS key,
+              CASE WHEN rolconnlimit <> -1 THEN rolconnlimit ELSE current_setting('max_connections')::INT - CASE WHEN rolsuper THEN 0 ELSE current_setting('superuser_reserved_connections')::INT END END AS connection_limit
+       FROM pg_roles
+       WHERE rolcanlogin}
+        },
+
+        database_size: {
+          prefix: %w(database),
+          query: %q{SELECT datname AS key, pg_database_size(oid) AS size FROM pg_database WHERE NOT datistemplate}
+        },
+
+        streaming_state: {
+          prefix: %w(streaming_state),
+          query: Gem::Version.new(server_version) >= Gem::Version.new('9.1') \
+          ? %q{SELECT CASE WHEN client_hostname IS NULL THEN 'socket' ELSE host(client_addr) END AS key,
+                    CASE state WHEN 'catchup' THEN 1 WHEN 'streaming' THEN 2 ELSE 0 END as value
+         FROM pg_stat_replication}
+          : nil
+        },
+
+        transactions: {
+          prefix: %w(database),
+          query: %q{SELECT dat.datname AS key, 'transactions' AS key2, xact_commit AS commit, xact_rollback AS rollback FROM pg_stat_database JOIN pg_database dat ON dat.oid = datid WHERE datallowconn AND NOT datistemplate}
+        },
+
+        xlog: {
+          prefix: %w(xlog),
+          query: Gem::Version.new(server_version) >= Gem::Version.new('9.0') \
+          ? %q{SELECT CASE WHEN pg_is_in_recovery() THEN NULL ELSE pg_current_xlog_location() END AS location,
+                pg_last_xlog_receive_location() AS receive_location,
+                pg_last_xlog_replay_location() AS replay_location}
+          : %q{SELECT pg_current_xlog_location() AS location}
+        }
+      }
+    end
+
+    def self.database_metrics(server_version)
+      {
+        Functions => {
+          prefix: %w(function),
+          query: Gem::Version.new(server_version) >= Gem::Version.new('8.4') \
+          ? %q{SELECT schemaname AS key,
+array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
+                      CASE WHEN pronargs = 0 THEN ''
+                      ELSE '-' || array_to_string(CASE WHEN pronargs > 16
+                                                    THEN ARRAY(SELECT args[i]
+                                                                 FROM generate_series(1, 8) AS _(i))
+                                                          || '-'::TEXT
+                                                          || ARRAY(SELECT args[i]
+                                                                     FROM generate_series(pronargs - 7, pronargs) AS _ (i))
+                                                          || funcid::TEXT
+                                                    ELSE args END, '-') END], '') AS key2,
+               calls, total_time, self_time
+  FROM (SELECT funcid, schemaname, funcname::TEXT, pronargs,
+               ARRAY(SELECT typname::TEXT
+                       FROM pg_type
+                       JOIN (SELECT args.i, proargtypes[args.i] AS typid
+                               FROM pg_catalog.generate_series(0, array_upper(proargtypes, 1)) AS args (i))
+                          AS args (i, typid) ON typid = pg_type.oid
+                       ORDER BY i) AS args,
+                      calls, total_time, self_time
+          FROM pg_stat_user_functions
+          JOIN pg_proc ON pg_proc.oid = funcid
+          WHERE schemaname NOT IN ('information_schema', 'pg_catalog')) AS funcs}
+          : nil
+        },
+
+        Locks => {
+          prefix: %w(table),
+          query: %q{SELECT nspname AS key,
+                         CASE rel.relkind WHEN 'r' THEN rel.relname ELSE crel.relname END AS key2,
+                         CASE rel.relkind WHEN 'r' THEN 'locks' ELSE 'index' END AS key3,
+                         CASE rel.relkind WHEN 'r' THEN mode ELSE rel.relname END AS key4,
+                         CASE rel.relkind WHEN 'r' THEN NULL ELSE 'locks' END AS key5,
+                         CASE rel.relkind WHEN 'r' THEN NULL ELSE mode END AS key6,
+                         count(*) AS value
+  FROM pg_locks
+  JOIN pg_database dat ON dat.oid = database
+  JOIN pg_class rel ON rel.oid = relation
+  LEFT JOIN pg_index ON indexrelid = rel.oid
+  LEFT JOIN pg_class crel ON indrelid = crel.oid
+  JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
+  WHERE locktype = 'relation' AND nspname <> 'pg_catalog' AND rel.relkind in ('r', 'i')
+  GROUP BY 1, 2, 3, 4, 5, 6}
+        },
+
+        TableSizes => {
+          prefix: %w(table),
+          query: %q{SELECT n.nspname AS key, r.relname AS key2,
+              pg_relation_size(r.oid) AS size,
+              pg_total_relation_size(r.oid) AS total_size
+       FROM pg_class r
+       JOIN pg_namespace n ON r.relnamespace = n.oid
+       WHERE r.relkind = 'r'
+         AND n.nspname NOT IN ('pg_catalog', 'information_schema')}
+        },
+
+        IndexSizes => {
+          prefix: %w(table),
+          query: %q{SELECT n.nspname AS key, cr.relname AS key2, 'index' AS key3,
+              ci.relname AS key4, pg_relation_size(ci.oid) AS size
+       FROM pg_class ci JOIN pg_index i ON ci.oid = i.indexrelid
+       JOIN pg_class cr ON cr.oid = i.indrelid
+       JOIN pg_namespace n on ci.relnamespace = n.oid
+       WHERE ci.relkind = 'i' AND cr.relkind = 'r'
+             AND n.nspname NOT IN ('pg_catalog', 'information_schema')}
+        },
+
+        TableStatio => {
+          prefix: %w(table),
+          query: %q{SELECT schemaname AS key, relname AS key2, 'statio' AS key3,
+              nullif(heap_blks_read, 0) AS heap_blks_read,
+              nullif(heap_blks_hit, 0) AS heap_blks_hit,
+              nullif(idx_blks_read, 0) AS idx_blks_read,
+              nullif(idx_blks_hit, 0) AS idx_blks_hit,
+              nullif(toast_blks_read, 0) AS toast_blks_read,
+              nullif(toast_blks_hit, 0) AS toast_blks_hit,
+              nullif(tidx_blks_read, 0) AS tidx_blks_read,
+              nullif(tidx_blks_hit, 0) AS tidx_blks_hit
+       FROM pg_statio_user_tables}
+        },
+
+        TableStats => {
+          prefix: %w(table),
+          query: Gem::Version.new(server_version) >= Gem::Version.new('9.1') \
+          ? %q{SELECT schemaname AS key, relname AS key2, 'stat' AS key3,
+              nullif(seq_scan, 0) AS seq_scan,
+              nullif(seq_tup_read, 0) AS seq_tup_read,
+              nullif(idx_scan, 0) AS idx_scan,
+              nullif(idx_tup_fetch, 0) AS idx_tup_fetch,
+              nullif(n_tup_ins, 0) AS n_tup_ins,
+              nullif(n_tup_upd, 0) AS n_tup_upd,
+              nullif(n_tup_del, 0) AS n_tup_del,
+              nullif(n_tup_hot_upd, 0) AS n_tup_hot_upd,
+              nullif(n_live_tup, 0) AS n_live_tup,
+              nullif(n_dead_tup, 0) AS n_dead_tup,
+              nullif(vacuum_count, 0) AS vacuum_count,
+              nullif(autovacuum_count, 0) AS autovacuum_count,
+              nullif(analyze_count, 0) AS analyze_count,
+              nullif(autoanalyze_count, 0) AS autoanalyze_count
+       FROM pg_stat_user_tables} \
+          : %q{SELECT schemaname AS key, relname AS key2, 'stat' AS key3,
+              nullif(seq_scan, 0) AS seq_scan,
+              nullif(seq_tup_read, 0) AS seq_tup_read,
+              nullif(idx_scan, 0) AS idx_scan,
+              nullif(idx_tup_fetch, 0) AS idx_tup_fetch,
+              nullif(n_tup_ins, 0) AS n_tup_ins,
+              nullif(n_tup_upd, 0) AS n_tup_upd,
+              nullif(n_tup_del, 0) AS n_tup_del,
+              nullif(n_tup_hot_upd, 0) AS n_tup_hot_upd,
+              nullif(n_live_tup, 0) AS n_live_tup,
+              nullif(n_dead_tup, 0) AS n_dead_tup
+       FROM pg_stat_user_tables},
+        },
+
+        IndexStatio => {
+          prefix: %w(table),
+          query: %q{SELECT schemaname AS key, relname AS key2, 'index' AS key3,
+                         indexrelname AS key4, 'statio' AS key5,
+                          nullif(idx_blks_read, 0) AS idx_blks_read,
+                          nullif(idx_blks_hit, 0) AS idx_blks_hit
+                    FROM pg_statio_user_indexes},
+        },
+
+        IndexStats => {
+          prefix: %w(table),
+          query: %q{SELECT schemaname AS key, relname AS key2, 'index' AS key3,
+                         indexrelname AS key4, 'stat' AS key5,
+                         nullif(idx_scan, 0) AS idx_scan,
+                         nullif(idx_tup_read, 0) AS idx_tup_read,
+                         nullif(idx_tup_fetch, 0) AS idx_tup_fetch
+         FROM pg_stat_user_indexes}
+        }
+      }
+    end
+  end
+end

data/lib/pg_metrics/pgbouncer_metrics.rb

@@ -0,0 +1,119 @@
+require "pg"
+
+module PgMetrics
+  module PgbouncerMetrics
+
+    def self.fetch_pgbouncer_metrics(app_name, conn_info)
+      metrics = []
+      conn = make_conn(conn_str(conn_info), app_name)
+      metrics = metrics.concat(fetch_stats_metrics(conn))
+      pool_results = fetch_pools(conn)
+      metrics = metrics.concat(extract_pool_metrics(pool_results))
+      database_results = fetch_databases(conn)
+      metrics = metrics.concat(extract_database_metrics(database_results))
+      metrics = metrics.concat(extract_backend_metrics(database_results, pool_results))
+      conn.finish
+      filter_metrics(metrics)
+    end
+
+    def self.make_conn(conn_str, app_name)
+      PG::Connection.new(conn_str)
+    end
+
+    def self.conn_str(conn_info, dbname = "pgbouncer", user = "admin")
+      [conn_info, %(dbname=#{dbname}), %(user=#{user})].join(" ")
+    end
+
+    def self.filter_metrics(metrics, regexp = nil)
+      metrics.reject! { |m| m[1].nil? }
+      metrics.reject! { |m| m[0].any? { |k| k =~ regexp  } } if regexp
+      metrics.inject([]) { |memo, m| memo << [sanitize_key(m[0]), m[1]] }
+    end
+
+    def self.fetch_stats_metrics(conn)
+      cols = %w(total_requests total_received total_sent total_query_time avg_req avg_recv avg_sent avg_query)
+      metrics = []
+      conn.exec("SHOW stats") do |results|
+        results.each do |tup|
+          cols.each do |col|
+            metrics << [["stats", tup["database"], col], tup[col]]
+          end
+        end
+      end
+      metrics
+    end
+
+    def self.fetch_pools(conn)
+      conn.exec("SHOW pools")
+    end
+
+    def self.extract_pool_metrics(results)
+      cols = %w(cl_active  cl_waiting sv_active sv_idle sv_used sv_tested sv_login maxwait)
+      results.inject([]) do |memo, tup|
+        cols.each do |col|
+          memo << [["pools", tup["database"], tup["user"], col], tup[col]]
+        end
+        memo
+      end
+    end
+
+    def self.fetch_databases(conn)
+      conn.exec("SHOW databases")
+    end
+
+    def self.extract_database_metrics(results)
+      cols = %w(pool_size reserve_pool)
+      results.inject([]) do |memo, tup|
+        cols.each do |col|
+          memo << [["databases", tup["name"], col], tup[col]]
+        end
+        memo
+      end
+    end
+
+    def self.extract_backend_metrics(database_results, pool_results)
+      databases = database_results.inject({}) do |memo, tup|
+        user = tup["force_user"].nil? ? :sameuser : tup["force_user"]
+        host = tup["host"].nil? ? "localhost" : tup["host"]
+        memo[tup["name"]] = {:host => host, :port => tup["port"], :database => tup["database"], :user => user}
+        memo
+      end
+
+      sum_cols = %w(cl_active cl_waiting sv_active sv_idle sv_used sv_tested sv_login)
+      max_cols = %w(max_wait)
+      cols = sum_cols.concat(max_cols)
+      sums = pool_results.inject({}) do |memo, tup|
+        database = databases[tup["database"]]
+        next memo if database.nil?
+        user = database[:user] === :sameuser ? tup["user"] : database[:user]
+        key = [database[:host], database[:port], database[:database], user]
+        vals = memo[key] || {
+          "cl_active" => 0,
+          "cl_waiting" => 0,
+          "sv_active" => 0,
+          "sv_idle" => 0,
+          "sv_used" => 0,
+          "sv_tested" => 0,
+          "sv_login" => 0,
+          "max_wait" => 0
+        }
+        sum_cols.each { |col| vals[col] += tup[col].to_i }
+        max_cols.each { |col| vals[col] = [tup[col].to_i, vals[col]].max }
+        memo[key] = vals
+        memo
+      end
+      sums.inject([]) do |memo, (key, val)|
+        cols.each do |col|
+          k = ["backends"] + key + [col]
+          memo << [k, val[col]]
+        end
+        memo
+      end
+    end
+
+    def self.sanitize_key(key)
+      key.inject([]) { |memo, el| memo << el.gsub(/[^-a-zA-Z_0-9]/, "_") }
+    end
+
+  end
+end

data/lib/pg_metrics/statsd.rb

@@ -0,0 +1,85 @@
+require 'optparse'
+require 'socket'
+require 'statsd-ruby'
+require 'set'
+
+module PgMetrics
+  module Statsd
+    APPNAME = "pg_metrics_statsd"
+
+    def self.main(args)
+      options = self.parse(args)
+
+      if options[:version]
+        STDOUT.puts %(pg_metrics #{PgMetrics::VERSION})
+        return 0
+      end
+
+      if options[:pgbouncer]
+        metrics = PgMetrics::PgbouncerMetrics::fetch_pgbouncer_metrics(APPNAME, options[:conn])
+      else
+        regexp = options[:exclude] ? options[:exclude] : nil
+
+        metrics = if options[:dbname]
+                    PgMetrics::Metrics::fetch_database_metrics(APPNAME, options[:conn], options[:dbname],
+                                                               options[:dbstats], regexp)
+                  else
+                    PgMetrics::Metrics::fetch_instance_metrics(APPNAME, options[:conn], regexp)
+                  end
+      end
+
+      statsd = ::Statsd.new(options[:host], options[:port]).tap do |sd|
+        sd.namespace = options[:scheme]
+      end
+
+      metrics.map! { |m| [m[0].join("."), m[1]] }
+
+      metrics.each { |m| STDOUT.puts m.join(" ") } if options[:verbose]
+
+      metrics.each do |m|
+        statsd.gauge(m[0], m[1])
+      end
+
+      exit 0
+    end
+
+    def self.parse(args)
+      options = {
+        host: "localhost",
+        port: 8125,
+        conn: "",
+        scheme: %(#{Socket.gethostname}.postgresql),
+        dbstats: [PgMetrics::Metrics::Functions,
+                  PgMetrics::Metrics::Locks,
+                  PgMetrics::Metrics::TableSizes,
+                  PgMetrics::Metrics::IndexSizes,
+                  PgMetrics::Metrics::TableStatio,
+                  PgMetrics::Metrics::TableStats,
+                  PgMetrics::Metrics::IndexStatio,
+                  PgMetrics::Metrics::IndexStats].to_set
+      }
+
+      OptionParser.new do |opts|
+        opts.on("-h", "--host STATSD_HOST", "StatsD host") { |v| options[:host] = v }
+        opts.on("-p", "--port STATSD_PORT", "StatsD port") { |v| options[:port] = v.to_i }
+        opts.on("-c", "--connection CONN", "PostgreSQL connection string") { |v| options[:conn] = v }
+        opts.on("-d", "--dbname DBNAME", "PostgreSQL database name for database metrics") { |v| options[:dbname] = v }
+        opts.on("-e", "--exclude REGEXP", "Exclude objects matching given regexp") { |v| options[:exclude] = ::Regexp.new(v) }
+        opts.on("-s", "--scheme SCHEME", "Metric namespace") { |v| options[:scheme] = v }
+        opts.on("--[no-]functions", "Collect database function stats") { |v| options[:dbstats].delete(PgMetrics::Metrics::Functions) unless v }
+        opts.on("--[no-]locks", "Collect database lock stats") { |v| options[:dbstats].delete(PgMetrics::Metrics::Locks) unless v }
+        opts.on("--[no-]table-sizes", "Collect database table size stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableSizes) unless v }
+        opts.on("--[no-]index-sizes", "Collect database index size stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexSizes) unless v }
+        opts.on("--[no-]table-statio", "Collect database table statio stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableStatio) unless v }
+        opts.on("--[no-]table-stats", "Collect database table stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableStats) unless v }
+        opts.on("--[no-]index-statio", "Collect database index statio stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexStatio) unless v }
+        opts.on("--[no-]index-stats", "Collect database index stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexStats) unless v }
+        opts.on("--pgbouncer", "Collect pgbouncer stats") { |v| options[:pgbouncer] = true }
+        opts.on("--verbose") { |v| options[:verbose] = true }
+        opts.on("--version") { |v| options[:version] = v }
+      end.order!(args)
+
+      options
+    end
+  end
+end

data/lib/pg_metrics/version.rb

@@ -0,0 +1,3 @@
+module PgMetrics
+  VERSION = "0.1.1"
+end

data/lib/pg_metrics.rb

@@ -0,0 +1,7 @@
+module PgMetrics
+end
+
+require 'pg_metrics/version'
+require 'pg_metrics/metrics'
+require 'pg_metrics/statsd'
+require 'pg_metrics/pgbouncer_metrics'

data/pg_metrics.gemspec

@@ -0,0 +1,38 @@
+lib = File.expand_path("../lib/", __FILE__)
+$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib)
+require "rake"
+require "pg_metrics/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "pg_metrics"
+  spec.version = PgMetrics::VERSION
+  spec.licenses = %w(MIT)
+  spec.date = "2015-03-17"
+  spec.summary = "pg_metrics"
+  spec.description = "PostgreSQL Metrics"
+  spec.authors = ["Michael Glaesemann"]
+  spec.email = ["michael.glaesemann@meetme.com"]
+  spec.files = FileList["{bin,lib,test}/**/*.*",
+                        "CHANGELOG.markdown",
+                        "DEV.markdown",
+                        "Gemfile",
+                        "Gemfile.lock",
+                        "LICENSE",
+                        "README.markdown",
+                        "pg_metrics.gemspec",
+                        "rakefile"].to_a
+  spec.executables = %w(pg_metrics_statsd)
+  spec.require_path = %(lib)
+  spec.test_files = FileList["test/**/*.*"].to_a
+  spec.extra_rdoc_files = %w(LICENSE README.markdown)
+  spec.homepage = "https://github.com/MeetMe/pg_metrics"
+  [["pg", ["~> 0.10"]],
+   ["statsd-ruby", ["~> 1.2", ">= 1.2.1"]]].each do |dep|
+    spec.add_runtime_dependency(*dep)
+  end
+
+  [["test-unit", [["~> 2.1", ">= 2.1.2.0"]]],
+   ["simplecov", ["~> 0.7", ">= 0.7.1"]]].each do |dep|
+    spec.add_development_dependency(*dep)
+  end
+end

data/rakefile

@@ -0,0 +1,8 @@
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.verbose = true
+end
+
+task :default => :test

data/test/helper.rb

@@ -0,0 +1,11 @@
+require "simplecov"
+SimpleCov.start { add_filter "/test/" }
+require "test/unit"
+
+[File.dirname(__FILE__),
+ File.join(File.dirname(__FILE__), "..", "lib")].each do |f|
+  $LOAD_PATH.unshift(f)
+end
+
+require "pg_metrics"
+

data/test/test_statsd.rb

@@ -0,0 +1,95 @@
+require "helper"
+require "set"
+
+module PgMetrics
+  module Test
+    class Statsd < ::Test::Unit::TestCase
+      def test_ok
+        assert(true)
+      end
+
+      def test_should_have_sensible_defaults
+        args = %w()
+        config = PgMetrics::Statsd::parse(args)
+        assert_equal("localhost", config[:host])
+        assert_equal(8125, config[:port])
+        assert_equal("", config[:conn])
+        assert_match(/\.postgresql$/, config[:scheme])
+      end
+
+      def test_should_set_host_and_port
+        args = %w(--host 127.0.0.1 --port 9000)
+        config = PgMetrics::Statsd::parse(args)
+        assert_equal("127.0.0.1", config[:host])
+        assert_equal(9000, config[:port])
+      end
+
+      def test_should_set_regexp_filter
+        args = %w(--exclude xdrop)
+        config = PgMetrics::Statsd::parse(args)
+        assert_equal(config[:exclude], ::Regexp.new(/xdrop/))
+      end
+
+      def test_should_set_connection
+        args = ["--connection", "host=localhost port=5493"]
+        config = PgMetrics::Statsd::parse(args)
+        assert_equal(config[:conn], "host=localhost port=5493")
+      end
+
+      def test_should_set_dbname
+        args = %w(--dbname prod)
+        config = PgMetrics::Statsd::parse(args)
+        assert_equal(config[:dbname], "prod")
+      end
+
+      def test_should_set_all_metrics
+        args = []
+        config = PgMetrics::Statsd::parse(args)
+        expected = [PgMetrics::Metrics::Functions,
+                    PgMetrics::Metrics::Locks,
+                    PgMetrics::Metrics::TableSizes,
+                    PgMetrics::Metrics::IndexSizes,
+                    PgMetrics::Metrics::TableStatio,
+                    PgMetrics::Metrics::TableStats,
+                    PgMetrics::Metrics::IndexStatio,
+                    PgMetrics::Metrics::IndexStats].to_set
+        assert_equal(config[:dbstats], expected)
+      end
+
+      def test_should_set_all_metrics_with_positive_locks
+        args = %w(--locks)
+        config = PgMetrics::Statsd::parse(args)
+        expected = [PgMetrics::Metrics::Functions,
+                    PgMetrics::Metrics::Locks,
+                    PgMetrics::Metrics::TableSizes,
+                    PgMetrics::Metrics::IndexSizes,
+                    PgMetrics::Metrics::TableStatio,
+                    PgMetrics::Metrics::TableStats,
+                    PgMetrics::Metrics::IndexStatio,
+                    PgMetrics::Metrics::IndexStats].to_set
+        assert_equal(config[:dbstats], expected)
+      end
+
+      def test_should_not_collect_locks
+        args = %w(--no-locks)
+        config = PgMetrics::Statsd::parse(args)
+        expected = [PgMetrics::Metrics::Functions,
+                    PgMetrics::Metrics::TableSizes,
+                    PgMetrics::Metrics::IndexSizes,
+                    PgMetrics::Metrics::TableStatio,
+                    PgMetrics::Metrics::TableStats,
+                    PgMetrics::Metrics::IndexStatio,
+                    PgMetrics::Metrics::IndexStats].to_set
+        assert_equal(config[:dbstats], expected)
+      end
+
+      def test_should_remove_all_but_locks
+        args = %w(--no-functions --no-table-sizes --no-index-sizes --no-table-statio --no-table-stats --no-index-stats --no-index-statio)
+        config = PgMetrics::Statsd::parse(args)
+        expected = [PgMetrics::Metrics::Locks].to_set
+        assert_equal(config[:dbstats], expected)
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: pg_metrics
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Michael Glaesemann
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: statsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.2.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+description: PostgreSQL Metrics
+email:
+- michael.glaesemann@meetme.com
+executables:
+- pg_metrics_statsd
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.markdown
+files:
+- lib/pg_metrics.rb
+- lib/pg_metrics/metrics.rb
+- lib/pg_metrics/pgbouncer_metrics.rb
+- lib/pg_metrics/statsd.rb
+- lib/pg_metrics/version.rb
+- test/helper.rb
+- test/test_statsd.rb
+- CHANGELOG.markdown
+- DEV.markdown
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.markdown
+- pg_metrics.gemspec
+- rakefile
+- bin/pg_metrics_statsd
+homepage: https://github.com/MeetMe/pg_metrics
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: pg_metrics
+test_files:
+- test/helper.rb
+- test/test_statsd.rb