heroku_mongo_watcher 0.0.1.alpha

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.idea/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in heroku_mongo_watcher.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/watcher

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require "heroku_mongo_watcher/cli"
+HerokuMongoWatcher::CLI.watch(*ARGV)

data/example/.watcher

@@ -0,0 +1,26 @@
+# text to single a error in your app -- will uptick w_err
+error_messages: [Error, Exception]
+
+# list of email addresses to notify upon an error
+notify: [bill@example.com, fred@example.com]
+
+# Currently we only support sending out emails if you have a gmail account
+# if either of these is blank, we will not send out an email
+gmail_username: bill@example.com
+gmail_password: sekret
+
+# interval in seconds to ping mongostat and to output and summarize data
+interval: 60
+
+# to connect to mongostat
+mongo_host: 'my-dbawesome-db.member0.mongolayer.com'
+mongo_username: 'ninja'
+mongo_password: 'skret'
+
+# to connect to heroku app
+# note you already need to have the heroku gem installed
+# with creditails already set up for this directory
+heroku_appname: 'my-heroku-app-name'
+
+#if you have multiple accounts
+heroku_account: 'my-heroku-account'

data/heroku_mongo_watcher.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "heroku_mongo_watcher/version"
+
+Gem::Specification.new do |s|
+  s.name        = "heroku_mongo_watcher"
+  s.version     = HerokuMongoWatcher::VERSION
+  s.authors     = ["Jonathan Tushman"]
+  s.email       = ["jtushman@gmail.com"]
+  s.homepage    = "http://www.tushman.com"
+  s.summary     = %q{Watches Mongostat and heroku logs to give you a pulse of your application}
+  s.description = %q{Also notifies you when certain thresholds are hit.  I have found this much more accurate than New Relic}
+
+  s.rubyforge_project = "heroku_mongo_watcher"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  s.add_runtime_dependency "term-ansicolor"
+  s.add_runtime_dependency "tlsmail"
+  s.add_runtime_dependency "heroku"
+end

data/lib/heroku_mongo_watcher/cli.rb

@@ -0,0 +1,309 @@
+require 'heroku_mongo_watcher/configuration'
+require 'term/ansicolor'
+require 'tlsmail'
+
+class HerokuMongoWatcher::CLI
+
+  extend HerokuMongoWatcher::Configuration
+
+  def self.watch(*args)
+    f = File.join(File.expand_path('~'),'.watcher')
+    configure_with(f)
+
+    # lock warnings flags
+    @lock_critical_notified = false
+    @lock_warning_notified = false
+
+    # average response time warning flags
+    @art_critical_notified = false
+    @art_warning_notified = false
+
+    # memos
+    @total_lines = 0
+    @total_service = 0
+    @total_wait = 0
+    @total_queue = 0
+    @total_router_errors = 0
+    @total_web_errors = 0
+    @max_service = 0
+    @dynos = 0
+    @slowest_request = nil
+    @errors = {}
+
+    @mutex = Mutex.new
+
+    # Call Tails heroku logs updates counts
+    heroku_watcher = Thread.new('heroku_logs') do
+      IO.popen("heroku logs --tail --app #{config[:heroku_appname]} --account #{config[:heroku_account]}") do |f|
+        while line = f.gets
+          process_heroku_router_line(line) if line.include? 'heroku[router]'
+          process_heroku_web_line(line) if line.include? 'app[web'
+        end
+      end
+    end
+
+    # Call Heroku PS to check the number of up dynos
+    heroku_ps = Thread.new('heroku_ps') do
+      while true do
+        dynos = 0
+        IO.popen("heroku ps --app #{config[:heroku_appname]} --account #{config[:heroku_account]}") do |p|
+          while line = p.gets
+            dynos += 1 if line =~ /^web/ && line.split(' ')[1] == 'up'
+          end
+        end
+        @mutex.synchronize { @dynos = dynos }
+        sleep(30)
+      end
+    end
+
+    puts
+    puts "|<---- heroku stats ------------------------------------------------------------>|<----mongo stats ------------------------------------------------------->|"
+    puts "dyno reqs       art    max    r_err w_err   wait  queue slowest                  | insert  query  update  faults locked qr|qw  netIn  netOut    time       |"
+
+    IO.popen("mongostat --rowcount 0 #{config[:interval]} --host #{config[:mongo_host]} --username #{config[:mongo_username]} --password #{config[:mongo_password]} --noheaders") do |f|
+      while line = f.gets
+        process_mongo_line(line) if line =~ /^/ && !(line =~ /^connected/)
+      end
+    end
+
+    heroku_watcher.join
+    heroku_ps.join
+
+  end
+
+  protected
+
+  def self.process_mongo_line(line)
+    items = line.split
+
+    inserts = items[0]
+    query = items[1]
+    update = items[2]
+    delete = items[3]
+    getmore = items[4]
+    command = items[5]
+    flushes = items[6]
+    mapped = items[7]
+    vsize = items[8]
+    res = items[9]
+    faults = items[10]
+    locked = items[11]
+    idx_miss = items[12]
+    qrw = items[13]
+    arw = items[14]
+    netIn = items[15]
+    netOut = items[16]
+    conn = items[17]
+    set = items[18]
+    repl = items[19]
+    time = items[20]
+
+    @mutex.synchronize do
+      art = average_response_time
+      err = @total_router_errors
+      average_wait = @total_lines > 0 ? @total_wait / @total_lines : 'N/A'
+      average_queue = @total_lines > 0 ? @total_queue / @total_lines : 'N/A'
+
+      print_errors
+
+      color_print @dynos, length: 4
+      color_print @total_lines
+      color_print art, warning: 1000, critical: 10_000, bold: true
+      color_print @max_service, warning: 10_000, critical: 20_000
+      color_print err, warning: 1, critical: 10
+      color_print @total_web_errors, warning: 1, critical: 10
+      color_print average_wait, warning: 10, critical: 100
+      color_print average_queue, warning: 10, critical: 100
+      color_print @slowest_request.slice(0,25), length: 28
+      print '|'
+      color_print inserts
+      color_print query
+      color_print update
+      color_print faults
+      color_print locked, bold: true, warning: 70, critical: 90
+      color_print qrw
+      color_print netIn
+      color_print netOut
+      color_print time, length: 10
+      printf "\n"
+
+      check_and_notify_locks(locked)
+      check_and_notify_response_time(art, @total_lines)
+
+      reset_memos
+
+    end
+
+
+  end
+
+  def self.average_response_time
+    @total_lines > 0 ? @total_service / @total_lines : 'N/A'
+  end
+
+  def self.print_errors
+    if @errors && @errors.keys && @errors.keys.length > 0
+      @errors.each do |error,count|
+        puts "\t\t[#{count}] #{error}"
+      end
+    end
+  end
+
+  def self.process_heroku_web_line(line)
+    # Only care about errors
+    error_messages = config[:error_messages]
+
+    if error_messages.any? { |mes| line.include? mes }
+      items = line.split
+      time = items[0]
+      process = items[1]
+      clean_line = line.sub(time,'').sub(process,'').strip
+      @mutex.synchronize do
+        @total_web_errors += 1
+        if @errors.has_key? clean_line
+          @errors[clean_line] = @errors[clean_line] + 1
+        else
+          @errors[clean_line] = 1
+        end
+      end
+    end
+
+
+
+  end
+
+  def self.process_heroku_router_line(line)
+    # 2012-07-05T20:24:10+00:00 heroku[router]: GET myapp.com/pxl/4fdbc97dc6b36c0030001160?value=1 dyno=web.14 queue=0 wait=0ms service=8ms status=200 bytes=35
+
+    # or if error
+
+    #2012-07-05T20:17:12+00:00 heroku[router]: Error H12 (Request timeout) -> GET myapp.com/crossdomain.xml dyno=web.4 queue= wait= service=30000ms status=503 bytes=0
+
+    items = line.split
+
+    if line =~ /Error/
+      @mutex.synchronize { @total_router_errors += 1 }
+    else
+
+
+      time = items[0]
+      process = items[1]
+      http_type = items[2]
+      url = items[3]
+      dyno = items[4].split('=').last if items[4]
+      queue = items[5].split('=').last.sub('ms', '') if items[5]
+      wait = items[6].split('=').last.sub('ms', '') if items[6]
+      service = items[7].split('=').last.sub('ms', '') if items[7]
+      status = items[8].split('=').last if items[8]
+      bytes = items[9].split('=').last if items[9]
+
+      if is_number?(service) && is_number?(wait) && is_number?(queue)
+        @mutex.synchronize do
+          @total_lines +=1
+          @total_service += Integer(service) if service
+          @total_wait += Integer(wait) if wait
+          @total_queue += Integer(queue) if queue
+          if Integer(service) > @max_service
+            @max_service = Integer(service)
+            @slowest_request = URI('http://' + url).path
+          end
+        end
+      end
+
+    end
+
+  end
+
+  def self.reset_memos
+    @total_service = @total_lines = @total_wait = @total_queue = @total_router_errors = @total_web_errors = @max_service = 0
+    @errors = {}
+  end
+
+  def self.check_and_notify_locks(locked)
+    l = Float(locked)
+    if l > 90
+      notify '[CRITICAL] Locks above 90%' unless @lock_critical_notified
+    elsif l > 70
+      notify '[WARNING] Locks above 70%' unless @lock_warning_notified
+    elsif l < 50
+      if @lock_warning_notified || @lock_critical_notified
+        notify '[Resolved] locks below 50%'
+        @lock_warning_notified = false
+        @lock_critical_notified = false
+      end
+
+    end
+  end
+
+  def self.check_and_notify_response_time(avt, requests)
+    return unless requests > 200
+    if avt > 10_000 || @total_router_errors > 100
+      notify "[SEVERE WARNING] Application not healthy | [#{@total_lines} rpm,#{avt} art]" unless @art_critical_notified
+      # @art_critical_notified = true
+    elsif avt > 500 || @total_router_errors > 10
+      notify "[WARNING] Application heating up | [#{@total_lines} rpm,#{avt} art]" unless @art_warning_notified
+      # @art_warning_notified = true
+    elsif avt < 300
+      if @art_warning_notified || @art_critical_notified
+        @art_warning_notified = false
+        @art_critical_notified = false
+      end
+    end
+  end
+
+  def self.notify(msg)
+    Thread.new('notify_admins') do
+      notify.each { |user_email| send_email(user_email, msg) } unless notify.empty?
+    end
+  end
+
+  def self.is_number?(string)
+    _is_number = true
+    begin
+      num = Integer(string)
+    rescue
+      _is_number = false
+    end
+    _is_number
+  end
+
+  def self.send_email(to, msg)
+
+    return unless config[:gmail_username] && config[:gmail_password]
+
+    content = [
+        "From: Mongo Watcher <#{config[:gmail_username]}>",
+        "To: #{to}",
+        "Subject: #{msg}",
+        "",
+        "RPM: #{@total_lines}",
+        "Average Reponse Time: #{average_response_time}",
+        "Application Errors: #{@total_web_errors}",
+        "Router Errors (timeouts): #{@total_router_errors}",
+        "Dynos: #{@dynos}"
+    ].join("\r\n")
+
+    Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
+    Net::SMTP.start('smtp.gmail.com', 587, 'gmail.com', config[:gmail_username], config[:gmail_password], :login) do |smtp|
+      smtp.send_message(content, config[:gmail_username], to)
+    end
+
+
+  end
+
+  def self.color_print(field, options ={})
+    options[:length] = 7 unless options[:length]
+    print Term::ANSIColor.bold if options[:bold] == true
+    if options[:critical] && is_number?(field) && Integer(field) > options[:critical]
+      print "\a" #beep
+      print Term::ANSIColor.red
+      print Term::ANSIColor.bold
+    elsif options[:warning] && is_number?(field) && Integer(field) > options[:warning]
+      print Term::ANSIColor.yellow
+    end
+    printf "%#{options[:length]}s", field
+    print Term::ANSIColor.clear
+  end
+
+
+end

data/lib/heroku_mongo_watcher/configuration.rb

@@ -0,0 +1,39 @@
+module HerokuMongoWatcher::Configuration
+
+  @@config = {
+      error_messages: ['Error', 'Exception', 'Cannot find impression', 'Timed out running'],
+      notify: [],
+      gmail_username: '',
+      gmail_password: '',
+      interval: 60,
+      mongo_host: '',
+      mongo_username: '',
+      mongo_password: '',
+      heroku_appname: '',
+      heroku_account: ''
+  }
+
+  @@valid_config_keys = @@config.keys
+
+  def config
+    @@config
+  end
+
+# Configure through yaml file
+  def configure_with(path_to_yaml_file)
+    begin
+      y_config = YAML::load(IO.read(path_to_yaml_file))
+    rescue Errno::ENOENT
+      log(:warning, "YAML configuration file couldn't be found. Using defaults."); return
+    rescue Psych::SyntaxError
+      log(:warning, "YAML configuration file contains invalid syntax. Using defaults."); return
+    end
+
+    configure(y_config)
+  end
+
+# Configure through hash
+  def configure(opts = {})
+    opts.each { |k, v| config[k.to_sym] = v if @@valid_config_keys.include? k.to_sym }
+  end
+end

data/lib/heroku_mongo_watcher/version.rb

@@ -0,0 +1,3 @@
+module HerokuMongoWatcher
+  VERSION = "0.0.1.alpha"
+end

data/lib/heroku_mongo_watcher.rb

@@ -0,0 +1,5 @@
+require "heroku_mongo_watcher/version"
+
+module HerokuMongoWatcher
+
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: heroku_mongo_watcher
+version: !ruby/object:Gem::Version
+  version: 0.0.1.alpha
+  prerelease: 6
+platform: ruby
+authors:
+- Jonathan Tushman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-20 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: term-ansicolor
+  requirement: &2157145900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157145900
+- !ruby/object:Gem::Dependency
+  name: tlsmail
+  requirement: &2157145160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157145160
+- !ruby/object:Gem::Dependency
+  name: heroku
+  requirement: &2157144040 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157144040
+description: Also notifies you when certain thresholds are hit.  I have found this
+  much more accurate than New Relic
+email:
+- jtushman@gmail.com
+executables:
+- watcher
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- bin/watcher
+- example/.watcher
+- heroku_mongo_watcher.gemspec
+- lib/heroku_mongo_watcher.rb
+- lib/heroku_mongo_watcher/cli.rb
+- lib/heroku_mongo_watcher/configuration.rb
+- lib/heroku_mongo_watcher/version.rb
+homepage: http://www.tushman.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: heroku_mongo_watcher
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: Watches Mongostat and heroku logs to give you a pulse of your application
+test_files: []