postgres-vacuum-monitor 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fd8fa8d98c1da93630ad6ca53b2d34417a34aab1
+  data.tar.gz: ec20fb7edead9610aea85944014df8e2ca41353b
+SHA512:
+  metadata.gz: acc26419c9fe7cc23583ade0813ae00e4b1025467c2bf7f12e8e252e329747c98a4b2435bc6503551dbc7c3cbecc8725ff8aeb77a49da275d21a4569ca40b5ce
+  data.tar.gz: 6e5fd0bc9126b1171acbd6604318c554a5199a58ff0fe83a99c136856ded8cc118a5c1c1c3bc4ef2a6ae2c004c306fce416c486e449bc44deea2f4e6934e2e58

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.iml
+.idea/
+Gemfile.lock
+log/*.log

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--format progress

data/.rubocop.yml

@@ -0,0 +1,5 @@
+inherit_gem:
+  salsify_rubocop: conf/rubocop_rails.yml
+
+AllCops:
+  TargetRubyVersion: 2.4

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+ - 2.4
+ - 2.5
+services:
+  - postgresql
+
+before_script:
+  - psql -c 'create database postgres_vacuum_monitor_test;' -U postgres
+
+addons:
+  postgresql: '9.6'

data/Appraisals

@@ -0,0 +1,3 @@
+appraise 'activerecord-5' do
+  gem 'activerecord', '>= 5.0.0', '< 6.0.0'
+end

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at fgarces@salsify.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in postgres-vacuum-monitor.gemspec
+#
+#
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2018 Fernando Garces
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+[![Build Status](https://travis-ci.org/salsify/postgres-vacuum-monitor.svg?branch=master)](https://travis-ci.org/salsify/postgres-vacuum-monitor)
+[![Maintainability](https://api.codeclimate.com/v1/badges/9ced178ca8fee231a935/maintainability)](https://codeclimate.com/github/salsify/postgres-vacuum-monitor/maintainability)
+[![Coverage Status](https://coveralls.io/repos/github/salsify/postgres-vacuum-monitor/badge.svg?branch=master)](https://coveralls.io/github/salsify/postgres-vacuum-monitor?branch=master)
+
+
+# Postgres::Vacuum::Monitor
+
+Postgres::Vacuum::Monitor provides queries that provide information about the number of dead tuples and long running queries.
+This information helps to diagnose and monitor two things: 
+1) That the current auto vacuum settings are working and keeping up.
+2) That there are no long running queries affecting the auto vacuuming daemon.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'postgres-vacuum-monitor'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install postgres-vacuum-monitor
+
+## Usage
+
+The job itself needs a class to report the information and can be configured by doing:
+
+```ruby
+Postgres::Vacuum::Monitor.configure do |config|
+  config.monitor_reporter_class_name = 'MetricsReporter'
+end
+```
+
+The class needs to follow this interface:
+```ruby
+class MetricsReporter
+  def report_event(name, attributes)
+  end
+end
+```
+
+For long running queries, the event name is `LongQueries` and the attributes are: 
+```ruby 
+{
+  start_time: # When the query started .
+  running_time: # How long has it been running in seconds.
+  application_name: # What's the application name that is running the query.
+  query: # The offending query.
+}
+```
+
+For auto vacuum the attributes are the following:
+
+```ruby 
+{
+  table: # Table name.
+  table_size: # How big is the table.
+  dead_tuples: # How many dead tuples are in the table.
+  tuples_over_limit: # How many dead tuples are over the auto vacuumer threshold.
+}
+```
+
+## New relic queries
+
+I use [New relic](https://rpm.newrelic.com) and use the following NRQL to create dashboards:
+
+#### Tuples over limit
+```SQL
+SELECT percentile(tuples_over_limit, 95) from AutoVacuumLagging facet table where appName = 'my-app' TIMESERIES 30 minutes since 1 day ago
+```
+
+#### Dead tuples
+```SQL
+SELECT percentile(dead_tuples) FROM AutoVacuumLagging facet table where appName = 'my-app' SINCE 1 DAY AGO TIMESERIES
+```     
+#### Long running queries
+```SQL
+SELECT application_name, query, running_time, start_time FROM LongQueries
+```
+
+#### Tables that need to be vacuumed
+```SQL
+SELECT uniques(table) FROM AutoVacuumLagging where appName = 'my-app' since 30 minutes ago
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/salsify/postgres-vacuum-monitor.

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'bundler/setup'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.verbose = false
+end
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "postgres/vacuum/monitor"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/postgres/vacuum/configuration.rb

@@ -0,0 +1,11 @@
+module Postgres
+  module Vacuum
+    class Configuration
+      attr_accessor :monitor_reporter_class_name
+
+      def initialize
+        @monitor_reporter_class_name = nil
+      end
+    end
+  end
+end

data/lib/postgres/vacuum/jobs/monitor_job.rb

@@ -0,0 +1,46 @@
+module Postgres
+  module Vacuum
+    module Jobs
+      class MonitorJob
+
+        AUTOVACUUM_QUERY_EVENT = 'AutoVacuumLagging'.freeze
+        LONG_QUERIES = 'LongQueries'.freeze
+
+        def perform(*)
+          ActiveRecord::Base.connection.execute(Postgres::Vacuum::Monitor::Query.long_running_queries).each do |row|
+            reporter_class.report_event(
+              LONG_QUERIES,
+              start_time: row['xact_start'],
+              running_time: row['seconds'],
+              application_name: row['application_name'],
+              query: row['query']
+            )
+          end
+
+          ActiveRecord::Base.connection.execute(Postgres::Vacuum::Monitor::Query.tables_eligible_vacuuming).each do |row|
+            reporter_class.report_event(
+              AUTOVACUUM_QUERY_EVENT,
+              table: row['relation'],
+              table_size: row['table_size'],
+              dead_tuples: row['dead_tuples'].to_i,
+              tuples_over_limit: row['dead_tuples'].to_i - row['autovacuum_vacuum_tuples'].to_i
+            )
+          end
+
+          true
+        end
+
+        def reporter_class
+          return @reporter_class_name if @reporter_class_name
+
+          @reporter_class_name = Postgres::Vacuum::Monitor.configuration.monitor_reporter_class_name&.safe_constantize
+          raise ConfigurationError.new('Missing or invalid report class name. Check your configuration') if @reporter_class_name.nil?
+
+          @reporter_class_name
+        end
+
+        ConfigurationError = Class.new(StandardError)
+      end
+    end
+  end
+end

data/lib/postgres/vacuum/monitor/query.rb

@@ -0,0 +1,70 @@
+module Postgres
+  module Vacuum
+    module Monitor
+      module Query
+        extend self
+
+        STATES = ["'idle in transaction'", "'active'"].freeze
+        TIME_LIMIT = 3600
+        THRESHOLD_SETTING = "'autovacuum_vacuum_threshold'".freeze
+        SCALE_FACTOR_SETTING = "'autovacuum_vacuum_scale_factor'".freeze
+        MAX_AGE_SETTING = "'autovacuum_freeze_max_age'".freeze
+        PG_CATALOG = "'pg_catalog'".freeze
+
+        def long_running_queries
+          <<-SQL
+            SELECT *
+            FROM (
+              SELECT
+                pid,
+                xact_start,
+                EXTRACT(EPOCH FROM (now() - xact_start)) AS seconds,
+                application_name,
+                query
+              FROM pg_stat_activity
+              WHERE state IN (#{STATES.join(', ')})
+              ORDER BY seconds DESC
+            ) AS long_queries
+            WHERE seconds > #{TIME_LIMIT};
+          SQL
+        end
+
+        def tables_eligible_vacuuming
+          # The query was taken from http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html
+          <<-SQL
+            WITH vbt AS (
+            SELECT setting AS autovacuum_vacuum_threshold FROM pg_settings WHERE name = #{THRESHOLD_SETTING}) , vsf AS (
+              SELECT setting AS autovacuum_vacuum_scale_factor FROM pg_settings WHERE name = #{SCALE_FACTOR_SETTING}) , fma AS (
+                SELECT setting AS autovacuum_freeze_max_age FROM pg_settings WHERE name = #{MAX_AGE_SETTING}) , sto AS (
+                  SELECT opt_oid, split_part(setting, '=', 1) AS param, split_part(setting, '=', 2) AS value FROM (
+                    SELECT oid opt_oid, unnest(reloptions) setting FROM pg_class
+                  ) opt
+                )
+                  SELECT '"'||ns.nspname||'"."'||c.relname||'"' AS relation,
+                          pg_size_pretty(pg_table_size(c.oid)) AS table_size,
+                          age(relfrozenxid) AS xid_age,
+                          coalesce(cfma.value::float, autovacuum_freeze_max_age::float) autovacuum_freeze_max_age,
+                          (
+                            coalesce(cvbt.value::float, autovacuum_vacuum_threshold::float) +
+                            coalesce(cvsf.value::float,autovacuum_vacuum_scale_factor::float) * pg_table_size(c.oid)
+                          ) AS autovacuum_vacuum_tuples,
+                          n_dead_tup AS dead_tuples
+                  FROM pg_class c
+                  JOIN pg_namespace ns ON ns.oid = c.relnamespace
+                  JOIN pg_stat_all_tables stat ON stat.relid = c.oid
+                  JOIN vbt ON (1=1) JOIN vsf ON (1=1)
+                  JOIN fma ON (1=1)
+                  LEFT JOIN sto cvbt ON cvbt.param = #{THRESHOLD_SETTING} AND c.oid = cvbt.opt_oid
+                  LEFT JOIN sto cvsf ON cvsf.param = #{SCALE_FACTOR_SETTING} AND c.oid = cvsf.opt_oid
+                  LEFT JOIN sto cfma ON cfma.param = #{MAX_AGE_SETTING} AND c.oid = cfma.opt_oid
+                  WHERE c.relkind = 'r'
+                    AND nspname <> #{PG_CATALOG}
+                    AND ( age(relfrozenxid) >= coalesce(cfma.value::float, autovacuum_freeze_max_age::float)
+                    OR coalesce(cvbt.value::float, autovacuum_vacuum_threshold::float) + coalesce(cvsf.value::float,autovacuum_vacuum_scale_factor::float) * pg_table_size(c.oid) <= n_dead_tup)
+            ORDER BY  age(relfrozenxid) DESC LIMIT 50;
+          SQL
+        end
+      end
+    end
+  end
+end

data/lib/postgres/vacuum/monitor/version.rb

@@ -0,0 +1,7 @@
+module Postgres
+  module Vacuum
+    module Monitor
+      VERSION = '0.2.2'.freeze
+    end
+  end
+end

data/lib/postgres/vacuum/monitor.rb

@@ -0,0 +1,27 @@
+require 'active_record'
+require 'postgres/vacuum/configuration'
+require 'postgres/vacuum/monitor/version'
+require 'postgres/vacuum/monitor/query'
+require 'postgres/vacuum/jobs/monitor_job'
+
+module Postgres
+  module Vacuum
+    module Monitor
+      class << self
+        attr_accessor :config
+      end
+
+      def self.configuration
+        @config ||= Configuration.new
+      end
+
+      def self.reset
+        @config = Configuration.new
+      end
+
+      def self.configure
+        yield(configuration)
+      end
+    end
+  end
+end

data/postgres-vacuum-monitor.gemspec

@@ -0,0 +1,35 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'postgres/vacuum/monitor/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'postgres-vacuum-monitor'
+  spec.version       = Postgres::Vacuum::Monitor::VERSION
+  spec.authors       = ['Fernando Garces']
+  spec.email         = ['fgarces@salsify.com']
+
+  spec.summary       = 'Simple stats collector for postgres auto vacuumer.'
+  spec.description   = 'Queries the Active Record DB for information regarding the auto vacuum process and long running queries.'
+  spec.homepage      = 'https://github.com/salsify/postgres-vacuum-monitor'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.4.0'
+
+  spec.add_development_dependency 'appraisal'
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'database_cleaner'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.2'
+  spec.add_development_dependency 'salsify_rubocop'
+
+  spec.add_dependency 'activerecord', '>= 5', '< 5.2'
+  spec.add_dependency 'pg', '~> 0.18'
+end

metadata

@@ -0,0 +1,194 @@
+--- !ruby/object:Gem::Specification
+name: postgres-vacuum-monitor
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+platform: ruby
+authors:
+- Fernando Garces
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-01-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: salsify_rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+description: Queries the Active Record DB for information regarding the auto vacuum
+  process and long running queries.
+email:
+- fgarces@salsify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Appraisals
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/postgres/vacuum/configuration.rb
+- lib/postgres/vacuum/jobs/monitor_job.rb
+- lib/postgres/vacuum/monitor.rb
+- lib/postgres/vacuum/monitor/query.rb
+- lib/postgres/vacuum/monitor/version.rb
+- postgres-vacuum-monitor.gemspec
+homepage: https://github.com/salsify/postgres-vacuum-monitor
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Simple stats collector for postgres auto vacuumer.
+test_files: []