heroku_mongo_watcher 0.0.2.alpha → 0.0.4.beta

data/README.md

@@ -1,11 +1,18 @@
 # Heroku Mongo Watcher
 
-I needed to find a way to marry mongo stats with my application stats
+Command line utility to monitor both your mongo and heroku instances, and to alert you when things are heating up
+
+## The Origin
+
+I have a pretty 'spiky' application that can go from having 10_000 requests per minute to 100_000, we need to notified
+when things are heating up so we can turn the appropriate dials.  We found new relic to be too slow
+(and not accurate enough once throughput levels got high), so we built this.
 
 It needed to accomplish the following:
 
-* See Mongostats and heroku stats at the same time, the key ones being requests per minute, average response time and lock %
-* Have multiple ways of notifying me: colors, beeps and email notifications
+* See Mongostats and heroku stats at the same time, the key ones being requests per minute, average response time
+  lock %, and error counts
+* Have multiple ways of notifying stake holders: colors, beeps and email notifications
 * Be able to parse the web log for certain errors and other logged events and aggregate data on them
 
 The output looks like the following ...
@@ -19,12 +26,44 @@ The output looks like the following ...
     		[11] Cannot find impression when looking for asset
       20  23935    190   7144      0     43      0      0            /crossdomain.xml|    307      0    618      1   21.6    0|0   260k   221k  15:05:19
 
+### Legend
+<table>
+    <tr><td>dynos</td><td>Number of running web instances</td></tr>
+    <tr><td>reqs</td><td>number of requests per sample</td></tr>
+    <tr><td>art</td><td>average request time</td></tr>
+    <tr><td>max</td><td>max request time</td></tr>
+    <tr><td>r_err</td><td>number of router errors, i.e. timeouts</td></tr>
+    <tr><td>w_err</td><td>number of web errros (see below)</td></tr>
+    <tr><td>wait</td><td>average router wait</td></tr>
+    <tr><td>queue</td><td>average router queue</td></tr>
+    <tr><td>slowest</td><td>path of the url that corresponds to the max request time</td></tr>
+    <tr><td>insert</td><td>number of mongo inserts</td></tr>
+    <tr><td>query</td><td>number of mongo queries</td></tr>
+    <tr><td>update</td><td>number of mongo updates</td></tr>
+    <tr><td>faults</td><td>number of mongo page faults</td></tr>
+    <tr><td>qr|qw</td><td>number of mongo's queued read and writes</td></tr>
+    <tr><td>netIn</td><td>size on mongo net in</td></tr>
+    <tr><td>netOut</td><td>size on mongo net out</td></tr>
+    <tr><td>time</td><td>the time sampled</td></tr>
+</table>
+
+### Web Errors (w_err)
+At least for me, one of the key features is aggregating signals from my web log (I look out for certain race conditions,
+and other errors).  You can can configure the `error_messages` array in your .watcher file to define which String we
+should report on.
+
+In concert with that is the `display_errors` configuration.  If set to true it will aggregate and display the errors
+found (see output above), set to false it will just put the total in the summary row
+
+
 ## Prereqs
 
 1. need to have a heroku account, and have the heroku gem running on your machine
+2. need to be able to run mongostat (e.q. has at least read-only admin access to your mongos)
 
 ## To install
 
 1. bundle install heroku_mongo_watcher
 2. create a .watcher file (see the examples) in your user directory ~/.watcher
 3. then run `bundle exec watcher`
+4. Ctrl-C out to quit

data/heroku_mongo_watcher.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.version     = HerokuMongoWatcher::VERSION
   s.authors     = ["Jonathan Tushman"]
   s.email       = ["jtushman@gmail.com"]
-  s.homepage    = "http://www.tushman.com"
+  s.homepage    = "https://github.com/jtushman/heroku_mongo_watcher"
   s.summary     = %q{Watches Mongostat and heroku logs to give you a pulse of your application}
   s.description = %q{Also notifies you when certain thresholds are hit.  I have found this much more accurate than New Relic}
 

data/lib/heroku_mongo_watcher/cli.rb

@@ -1,5 +1,5 @@
 require 'heroku_mongo_watcher/configuration'
-require 'term/ansicolor'
+require 'heroku_mongo_watcher/data_row'
 
 #http://stackoverflow.com/a/9117903/192791
 require 'net/smtp'
@@ -16,11 +16,13 @@ require 'tlsmail'
 
 class HerokuMongoWatcher::CLI
 
-  extend HerokuMongoWatcher::Configuration
+  def self.config
+    HerokuMongoWatcher::Configuration.instance.config
+  end
 
   def self.watch(*args)
-    f = File.join(File.expand_path('~'),'.watcher')
-    configure_with(f)
+
+    notify("Mongo Watcher enabled!")
 
     # lock warnings flags
     @lock_critical_notified = false
@@ -30,17 +32,8 @@ class HerokuMongoWatcher::CLI
     @art_critical_notified = false
     @art_warning_notified = false
 
-    # memos
-    @total_lines = 0
-    @total_service = 0
-    @total_wait = 0
-    @total_queue = 0
-    @total_router_errors = 0
-    @total_web_errors = 0
-    @max_service = 0
-    @dynos = 0
-    @slowest_request = nil
-    @errors = {}
+    @current_row = HerokuMongoWatcher::DataRow.new
+    @last_row = HerokuMongoWatcher::DataRow.new
 
     @mutex = Mutex.new
 
@@ -48,8 +41,10 @@ class HerokuMongoWatcher::CLI
     heroku_watcher = Thread.new('heroku_logs') do
       IO.popen("heroku logs --tail --app #{config[:heroku_appname]} --account #{config[:heroku_account]}") do |f|
         while line = f.gets
-          process_heroku_router_line(line) if line.include? 'heroku[router]'
-          process_heroku_web_line(line) if line.include? 'app[web'
+          @mutex.synchronize do
+            @current_row.process_heroku_router_line(line) if line.include? 'heroku[router]'
+            @current_row.process_heroku_web_line(line) if line.include? 'app[web'
+          end
         end
       end
     end
@@ -63,176 +58,42 @@ class HerokuMongoWatcher::CLI
             dynos += 1 if line =~ /^web/ && line.split(' ')[1] == 'up'
           end
         end
-        @mutex.synchronize { @dynos = dynos }
+        @mutex.synchronize { @current_row.dynos = dynos }
         sleep(30)
       end
     end
 
-    puts
-    puts "|<---- heroku stats ------------------------------------------------------------>|<----mongo stats ------------------------------------------------------->|"
-    puts "dyno reqs       art    max    r_err w_err   wait  queue slowest                  | insert  query  update  faults locked qr|qw  netIn  netOut    time       |"
+
+    HerokuMongoWatcher::DataRow.print_header
 
     IO.popen("mongostat --rowcount 0 #{config[:interval]} --host #{config[:mongo_host]} --username #{config[:mongo_username]} --password #{config[:mongo_password]} --noheaders") do |f|
       while line = f.gets
-        process_mongo_line(line) if line =~ /^/ && !(line =~ /^connected/)
-      end
-    end
-
-    heroku_watcher.join
-    heroku_ps.join
-
-  end
-
-  protected
-
-  def self.process_mongo_line(line)
-    items = line.split
-
-    inserts = items[0]
-    query = items[1]
-    update = items[2]
-    delete = items[3]
-    getmore = items[4]
-    command = items[5]
-    flushes = items[6]
-    mapped = items[7]
-    vsize = items[8]
-    res = items[9]
-    faults = items[10]
-    locked = items[11]
-    idx_miss = items[12]
-    qrw = items[13]
-    arw = items[14]
-    netIn = items[15]
-    netOut = items[16]
-    conn = items[17]
-    set = items[18]
-    repl = items[19]
-    time = items[20]
-
-    @mutex.synchronize do
-      art = average_response_time
-      err = @total_router_errors
-      average_wait = @total_lines > 0 ? @total_wait / @total_lines : 'N/A'
-      average_queue = @total_lines > 0 ? @total_queue / @total_lines : 'N/A'
-
-      print_errors
-
-      color_print @dynos, length: 4
-      color_print @total_lines
-      color_print art, warning: 1000, critical: 10_000, bold: true
-      color_print @max_service, warning: 10_000, critical: 20_000
-      color_print err, warning: 1, critical: 10
-      color_print @total_web_errors, warning: 1, critical: 10
-      color_print average_wait, warning: 10, critical: 100
-      color_print average_queue, warning: 10, critical: 100
-      color_print @slowest_request.slice(0,25), length: 28
-      print '|'
-      color_print inserts
-      color_print query
-      color_print update
-      color_print faults
-      color_print locked, bold: true, warning: 70, critical: 90
-      color_print qrw
-      color_print netIn
-      color_print netOut
-      color_print time, length: 10
-      printf "\n"
-
-      check_and_notify_locks(locked)
-      check_and_notify_response_time(art, @total_lines)
-
-      reset_memos
-
-    end
-
+        next unless line =~ /^/ && !(line =~ /^connected/)
+        @mutex.synchronize do
+          @current_row.process_mongo_line(line)
 
-  end
+          @current_row.print_row
 
-  def self.average_response_time
-    @total_lines > 0 ? @total_service / @total_lines : 'N/A'
-  end
-
-  def self.print_errors
-    if config[:print_errors] &&@errors && @errors.keys && @errors.keys.length > 0
-      @errors.each do |error,count|
-        puts "\t\t[#{count}] #{error}"
-      end
-    end
-  end
+          check_and_notify_locks
+          check_and_notify_response_time
 
-  def self.process_heroku_web_line(line)
-    # Only care about errors
-    error_messages = config[:error_messages]
+          @last_row = @current_row
+          @current_row = HerokuMongoWatcher::DataRow.new
+          @current_row.dynos = @last_row.dynos
 
-    if error_messages.any? { |mes| line.include? mes }
-      items = line.split
-      time = items[0]
-      process = items[1]
-      clean_line = line.sub(time,'').sub(process,'').strip
-      @mutex.synchronize do
-        @total_web_errors += 1
-        if @errors.has_key? clean_line
-          @errors[clean_line] = @errors[clean_line] + 1
-        else
-          @errors[clean_line] = 1
         end
       end
     end
 
-
-
-  end
-
-  def self.process_heroku_router_line(line)
-    # 2012-07-05T20:24:10+00:00 heroku[router]: GET myapp.com/pxl/4fdbc97dc6b36c0030001160?value=1 dyno=web.14 queue=0 wait=0ms service=8ms status=200 bytes=35
-
-    # or if error
-
-    #2012-07-05T20:17:12+00:00 heroku[router]: Error H12 (Request timeout) -> GET myapp.com/crossdomain.xml dyno=web.4 queue= wait= service=30000ms status=503 bytes=0
-
-    items = line.split
-
-    if line =~ /Error/
-      @mutex.synchronize { @total_router_errors += 1 }
-    else
-
-
-      time = items[0]
-      process = items[1]
-      http_type = items[2]
-      url = items[3]
-      dyno = items[4].split('=').last if items[4]
-      queue = items[5].split('=').last.sub('ms', '') if items[5]
-      wait = items[6].split('=').last.sub('ms', '') if items[6]
-      service = items[7].split('=').last.sub('ms', '') if items[7]
-      status = items[8].split('=').last if items[8]
-      bytes = items[9].split('=').last if items[9]
-
-      if is_number?(service) && is_number?(wait) && is_number?(queue)
-        @mutex.synchronize do
-          @total_lines +=1
-          @total_service += Integer(service) if service
-          @total_wait += Integer(wait) if wait
-          @total_queue += Integer(queue) if queue
-          if Integer(service) > @max_service
-            @max_service = Integer(service)
-            @slowest_request = URI('http://' + url).path
-          end
-        end
-      end
-
-    end
+    heroku_watcher.join
+    heroku_ps.join
 
   end
 
-  def self.reset_memos
-    @total_service = @total_lines = @total_wait = @total_queue = @total_router_errors = @total_web_errors = @max_service = 0
-    @errors = {}
-  end
+  protected
 
-  def self.check_and_notify_locks(locked)
-    l = Float(locked)
+  def self.check_and_notify_locks
+    l = Float(@current_row.lock)
     if l > 90
       notify '[CRITICAL] Locks above 90%' unless @lock_critical_notified
     elsif l > 70
@@ -247,15 +108,15 @@ class HerokuMongoWatcher::CLI
     end
   end
 
-  def self.check_and_notify_response_time(avt, requests)
-    return unless requests > 200
-    if avt > 10_000 || @total_router_errors > 100
-      notify "[SEVERE WARNING] Application not healthy | [#{@total_lines} rpm,#{avt} art]" unless @art_critical_notified
+  def self.check_and_notify_response_time
+    return unless @current_row.total_requests > 200
+    if @current_row.average_response_time > 10_000 || @current_row.total_router_errors > 100
+      notify "[SEVERE WARNING] Application not healthy | [#{@current_row.total_requests} rpm,#{@current_row.average_response_time} art]" unless @art_critical_notified
       # @art_critical_notified = true
-    elsif avt > 500 || @total_router_errors > 10
-      notify "[WARNING] Application heating up | [#{@total_lines} rpm,#{avt} art]" unless @art_warning_notified
+    elsif @current_row.average_response_time > 500 || @current_row.total_router_errors > 10 || @current_row.total_requests > 30_000
+      notify "[WARNING] Application heating up | [#{@current_row.total_requests} rpm,#{@current_row.average_response_time} art]" unless @art_warning_notified
       # @art_warning_notified = true
-    elsif avt < 300
+    elsif @current_row.average_response_time < 300
       if @art_warning_notified || @art_critical_notified
         @art_warning_notified = false
         @art_critical_notified = false
@@ -270,16 +131,6 @@ class HerokuMongoWatcher::CLI
     end
   end
 
-  def self.is_number?(string)
-    _is_number = true
-    begin
-      num = Integer(string)
-    rescue
-      _is_number = false
-    end
-    _is_number
-  end
-
   def self.send_email(to, msg)
     return unless config[:gmail_username] && config[:gmail_password]
     content = [
@@ -287,11 +138,11 @@ class HerokuMongoWatcher::CLI
         "To: #{to}",
         "Subject: #{msg}",
         "",
-        "RPM: #{@total_lines}",
-        "Average Reponse Time: #{average_response_time}",
-        "Application Errors: #{@total_web_errors}",
-        "Router Errors (timeouts): #{@total_router_errors}",
-        "Dynos: #{@dynos}"
+        "RPM: #{@last_row.total_requests}",
+        "Average Reponse Time: #{@last_row.average_response_time}",
+        "Application Errors: #{@last_row.total_web_errors}",
+        "Router Errors (timeouts): #{@last_row.total_router_errors}",
+        "Dynos: #{@last_row.dynos}"
     ].join("\r\n")
 
     Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
@@ -300,19 +151,4 @@ class HerokuMongoWatcher::CLI
     end
   end
 
-  def self.color_print(field, options ={})
-    options[:length] = 7 unless options[:length]
-    print Term::ANSIColor.bold if options[:bold] == true
-    if options[:critical] && is_number?(field) && Integer(field) > options[:critical]
-      print "\a" #beep
-      print Term::ANSIColor.red
-      print Term::ANSIColor.bold
-    elsif options[:warning] && is_number?(field) && Integer(field) > options[:warning]
-      print Term::ANSIColor.yellow
-    end
-    printf "%#{options[:length]}s", field
-    print Term::ANSIColor.clear
-  end
-
-
 end

data/lib/heroku_mongo_watcher/configuration.rb

@@ -1,4 +1,7 @@
-module HerokuMongoWatcher::Configuration
+require 'singleton'
+
+class HerokuMongoWatcher::Configuration
+  include Singleton
 
   @@config = {
       error_messages: ['Error', 'Exception', 'Cannot find impression', 'Timed out running'],
@@ -11,11 +14,16 @@ module HerokuMongoWatcher::Configuration
       mongo_password: '',
       heroku_appname: '',
       heroku_account: '',
-      display_errors: true
+      print_errors: true
   }
 
   @@valid_config_keys = @@config.keys
 
+  def initialize
+    f = File.join(File.expand_path('~'),'.watcher')
+    configure_with(f)
+  end
+
   def config
     @@config
   end

data/lib/heroku_mongo_watcher/data_row.rb

@@ -0,0 +1,188 @@
+require 'term/ansicolor'
+require 'heroku_mongo_watcher/configuration'
+class HerokuMongoWatcher::DataRow
+
+  # Heroku Attributes
+  @@attributes = [:total_requests, :total_service, :total_wait,
+                  :total_queue, :total_router_errors, :total_web_errors,
+                  :max_service, :slowest_request, :errors, :dynos]
+
+  # Mongo Attributes
+  @@attributes.concat [:inserts, :queries, :ops, :updates, :deletes,
+                       :faults, :lock, :qrw, :net_in, :net_out]
+
+  @@attributes.each { |attr| attr_accessor attr }
+
+  def initialize
+    @@attributes.each { |attr| send "#{attr}=", 0 }
+    self.slowest_request = nil
+    self.errors = {}
+  end
+
+  def config
+    HerokuMongoWatcher::Configuration.instance.config
+  end
+
+  def average_response_time
+    total_requests > 0 ? total_service / total_requests : 'N/A'
+  end
+
+  def average_wait
+    total_requests > 0 ? total_wait / total_requests : 'N/A'
+  end
+
+  def average_queue
+    total_requests > 0 ? total_queue / total_requests : 'N/A'
+  end
+
+  def process_heroku_router_line(line)
+    items = line.split
+
+    if line =~ /Error/
+      # Note: The lion share of these are timeouts
+      #Full list here: https://devcenter.heroku.com/articles/error-codes
+      self.total_router_errors += 1
+    else
+      time = items[0]
+      process = items[1]
+      http_type = items[2]
+      url = items[3]
+      dyno = items[4].split('=').last if items[4]
+      queue = items[5].split('=').last.sub('ms', '') if items[5]
+      wait = items[6].split('=').last.sub('ms', '') if items[6]
+      service = items[7].split('=').last.sub('ms', '') if items[7]
+      status = items[8].split('=').last if items[8]
+      bytes = items[9].split('=').last if items[9]
+
+      if is_number?(service) && is_number?(wait) && is_number?(queue)
+        self.total_requests +=1
+        self.total_service += Integer(service) if service
+        self.total_wait += Integer(wait) if wait
+        self.total_queue += Integer(queue) if queue
+        if Integer(service) > self.max_service
+          self.max_service = Integer(service)
+          self.slowest_request = URI('http://' + url).path
+        end
+      end
+
+    end
+  end
+
+  def process_heroku_web_line(line)
+    # Only care about errors
+    error_messages = config[:error_messages]
+
+    if error_messages.any? { |mes| line.include? mes }
+      items = line.split
+      time = items[0]
+      process = items[1]
+      clean_line = line.sub(time, '').sub(process, '').strip
+
+      self.total_web_errors += 1
+      if self.errors.has_key? clean_line
+        self.errors[clean_line] = self.errors[clean_line] + 1
+      else
+        self.errors[clean_line] = 1
+      end
+    end
+
+  end
+
+  def process_mongo_line(line)
+    items = line.split
+
+    @inserts = items[0]
+    @queries = items[1]
+    @updates = items[2]
+    delete = items[3]
+    getmore = items[4]
+    command = items[5]
+    flushes = items[6]
+    mapped = items[7]
+    vsize = items[8]
+    res = items[9]
+    @faults = items[10]
+    @locked = items[11]
+    idx_miss = items[12]
+    @qrw = items[13]
+    arw = items[14]
+    @net_in = items[15]
+    @net_out = items[16]
+    conn = items[17]
+    set = items[18]
+    repl = items[19]
+    @mongo_time = items[20]
+
+  end
+
+  def self.print_header
+    puts
+    puts "|<---- heroku stats ------------------------------------------------------------>|<----mongo stats ------------------------------------------------------->|"
+    puts "dyno reqs       art    max    r_err w_err   wait  queue slowest                  | insert  query  update  faults locked qr|qw  netIn  netOut    time       |"
+  end
+
+  def print_row
+    print_errors
+
+    color_print @dynos, length: 4
+    color_print @total_requests, warning: 30_000, critical: 50_000
+    color_print average_response_time, warning: 1000, critical: 10_000, bold: true
+    color_print @max_service, warning: 20_000, critical: 27_000
+    color_print @total_router_errors, warning: 1, critical: 10
+    color_print @total_web_errors, warning: 1, critical: 10
+    color_print average_wait, warning: 10, critical: 100
+    color_print average_queue, warning: 10, critical: 100
+    color_print @slowest_request, length: 28, slice: 25
+    print '|'
+    color_print @inserts
+    color_print @queries
+    color_print @updates
+    color_print @faults
+    color_print @locked, bold: true, warning: 40, critical: 70
+    color_print @qrw
+    color_print @net_in
+    color_print @net_out
+    color_print @mongo_time, length: 10
+    printf "\n"
+  end
+
+  def print_errors
+    if config[:print_errors] && @errors && @errors.keys && @errors.keys.length > 0
+      @errors.each do |error,count|
+        puts "\t\t[#{count}] #{error}"
+      end
+    end
+  end
+
+  private
+
+  def is_number?(string)
+    _is_number = true
+    begin
+      num = Integer(string)
+    rescue
+      _is_number = false
+    end
+    _is_number
+  end
+
+  def color_print(field, options ={})
+    options[:length] = 7 unless options[:length]
+    print Term::ANSIColor.bold if options[:bold] == true
+    if options[:critical] && is_number?(field) && Integer(field) > options[:critical]
+      print "\a" #beep
+      print Term::ANSIColor.red
+      print Term::ANSIColor.bold
+    elsif options[:warning] && is_number?(field) && Integer(field) > options[:warning]
+      print Term::ANSIColor.yellow
+    end
+
+    str = field.to_s
+    str = str.slice(0, options[:slice]) if options[:slice] && str && str.length > 0
+
+    printf "%#{options[:length]}s", str
+    print Term::ANSIColor.clear
+  end
+
+
+end

data/lib/heroku_mongo_watcher/version.rb

@@ -1,3 +1,3 @@
 module HerokuMongoWatcher
-  VERSION = "0.0.2.alpha"
+  VERSION = "0.0.4.beta"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: heroku_mongo_watcher
 version: !ruby/object:Gem::Version
-  version: 0.0.2.alpha
+  version: 0.0.4.beta
   prerelease: 6
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-20 00:00:00.000000000Z
+date: 2012-07-23 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
-  requirement: &2157594420 !ruby/object:Gem::Requirement
+  requirement: &2157942700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157594420
+  version_requirements: *2157942700
 - !ruby/object:Gem::Dependency
   name: tlsmail
-  requirement: &2157593520 !ruby/object:Gem::Requirement
+  requirement: &2157942040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157593520
+  version_requirements: *2157942040
 - !ruby/object:Gem::Dependency
   name: heroku
-  requirement: &2157586200 !ruby/object:Gem::Requirement
+  requirement: &2157941260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157586200
+  version_requirements: *2157941260
 description: Also notifies you when certain thresholds are hit.  I have found this
   much more accurate than New Relic
 email:
@@ -63,8 +63,9 @@ files:
 - lib/heroku_mongo_watcher.rb
 - lib/heroku_mongo_watcher/cli.rb
 - lib/heroku_mongo_watcher/configuration.rb
+- lib/heroku_mongo_watcher/data_row.rb
 - lib/heroku_mongo_watcher/version.rb
-homepage: http://www.tushman.com
+homepage: https://github.com/jtushman/heroku_mongo_watcher
 licenses: []
 post_install_message: 
 rdoc_options: []