panoptimon 0.0.2

data/panoptimon.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/panoptimon/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Eric Wilhelm"]
+  gem.description   = %q{The All-Seeing System Monitor Daemon}
+  gem.summary       = %q{Panoptimon collects and routes system metrics.}
+
+  gem.email         = "sysops@sourcefire.com"
+  gem.homepage      = "https://github.com/synthesist/panoptimon"
+
+  gem.license       = 'bsd' # The (three-clause) BSD License
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "panoptimon"
+  gem.require_paths = ["lib"]
+  gem.version       = Panoptimon::VERSION
+  gem.required_ruby_version = '>= 1.9'
+  # Core gem dependencies
+  gem.add_dependency 'eventmachine', '~> 1.0.0'
+  gem.add_dependency 'daemons'
+  gem.add_dependency 'json'
+  gem.add_dependency 'thin', '~> 1.5.0'
+
+  # Plugin gem dependencies
+  gem.add_dependency 'riemann-client'
+
+  # Collector gem dependencies
+  gem.add_dependency 'sys-filesystem'
+  gem.add_dependency 'mysql'
+end

data/plugins/daemon_health/README.md

@@ -0,0 +1,31 @@
+# Daemon Health Plugin (Self-Happiness Metrics Collector)
+
+This plugin will periodically add metrics to the bus, while also
+listening to other metrics to generate statics about the collected data.
+
+# Configuration
+
+`periods = ["60m", 300, 30]` - an array of reporting periods (in
+seconds, or minutes with 'm' suffix)
+
+`interval = 30` - the reporting interval (and rollup granularity) in
+seconds.  The periods should all be a multiple of this for consistent
+results.
+
+
+```json
+  {
+   "daemon_health|300|metrics|nps"  : 4.5867, # 5m moving avg
+   "daemon_health|900|metrics|nps"  : 5.3778, # 15m ...
+   "daemon_health|3600|metrics|nps" : 5.4373, # 1hr...
+   "daemon_health|collectors"       : 8,
+   "daemon_health|loaded_plugins"   : 5,
+   "daemon_health|active_plugins"   : 3,      # with callbacks
+   "daemon_health|uptime"           : 960,    # seconds
+   "daemon_health|rss"              : 21784,  # kB
+  }
+```
+
+Moving averages are calculated within the configured window (unless the
+uptime is less than the window, in which case it is the average over
+uptime.)

data/plugins/daemon_health/daemon_health.json

@@ -0,0 +1,4 @@
+{
+  "interval" : 10,
+  "periods" : [900,60,20,10]
+}

data/plugins/daemon_health/daemon_health.rb

@@ -0,0 +1,34 @@
+
+require 'rusage'
+$LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
+require 'panoptimon-plugin-daemon_health/rollup'
+
+config[:interval] ||= 60
+
+start = then_ = Time.now
+rolling = Rollup.new(start, config) # brief history
+
+setup = ->() {
+EM.add_periodic_timer(config[:interval], ->(){
+  now = Time.now
+  elapsed = now - then_
+
+  metrics = rolling.roll(now).merge({
+    'uptime' => (now - start).round(0),
+    'rss'    => Process.rusage.maxrss,
+    collectors:     monitor.collectors.length,
+    active_plugins: monitor.plugins.keys.length,
+    loaded_plugins: monitor.loaded_plugins.keys.length,
+  })
+  m = Metric.new('daemon_health', metrics)
+  monitor.bus.notify(m)
+
+  then_ = now
+})
+}
+
+return ->(metric) {
+  return if metric.has_key?('daemon_health|uptime') # skip our own data
+  if setup; setup[] ; setup = nil; end
+  rolling.log(metrics: metric.keys.length)
+}

data/plugins/daemon_health/lib/panoptimon-plugin-daemon_health/rollup.rb

@@ -0,0 +1,64 @@
+class Rollup < Array
+
+def initialize(start, config)
+  @start = start
+  @start ||= Time.now
+  @periods = (config[:periods] || %w{60m 15m 5m 1m}).map {|m|
+    m.is_a?(String) ? m.sub(/m$/, '') ? m.to_i*60 : m.to_i : m
+  }.sort {|a,b| b <=> a}
+  _reset_current(@start)
+end
+
+def log (h)
+  h.each {|k,v|
+    c = @current[1][k] ||= {min: nil, max: nil, sum: 0}
+    c[:sum] += v
+    c[:min] ||= v; c[:min] = v if v < c[:min]
+    c[:max] ||= v; c[:max] = v if v > c[:max]
+  }
+end
+
+def sunset(horizon)
+  cut_here = -1
+  self.each {|x| break if x[0] > horizon; cut_here += 1 }
+  self.slice!(0..cut_here) if cut_here >= 0
+end
+
+def _reset_current (time); @current = [time, {}]; end
+def _shortroll(now)
+  c = @current
+  elapsed = now - c[0]
+  self.push(c)
+  _reset_current(now)
+end
+
+def roll(now)
+  self.sunset(now - @periods[0])
+  self._shortroll(now)
+
+  agg = Hash[@periods.map {|p| [p, {}]}]
+  self.each {|x|
+    age = now - x[0]
+    thru = (0...@periods.length).to_a.find {|i| age > @periods[i]} ||
+      @periods.length
+    @periods[0...thru].each {|p|
+      x[1].each {|k,v|
+        o = agg[p][k] ||= {
+          # min: nil, max: nil, # XXX not sure if we need that here
+          sum: 0}
+        o[:sum] += v[:sum]
+        # o[:min] ||= v[:min]; o[:min] = v[:min] if v[:min] < o[:min]
+        # o[:max] ||= v[:max]; o[:max] = v[:max] if v[:max] > o[:max]
+      }
+    }
+  }
+  up = now - @start
+  return Hash[agg.map {|p,h|
+    span = up > p ? p : up
+    h.each {|k,v| v[:nps] = (v.delete(:sum).to_f / span).round(4)}
+    [p.to_s, h]
+  }]
+  
+end
+
+end

data/plugins/daemon_health/panoptimon-plugin-daemon_health.gemspec

@@ -0,0 +1,10 @@
+Gem::Specification.new { |s|
+  s.name = 'panoptimon-plugin-daemon_health'
+  s.version = '0.0.1'
+  s.summary = 'report internal daemon metrics'
+  s.authors = ['Eric Wilhelm']
+
+  s.files         = `git ls-files`.split($\)
+
+  s.add_dependency 'rusage'
+}

data/plugins/daemon_health/spec/moving_avg_spec.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require 'rspec'
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+require 'panoptimon-plugin-daemon_health/rollup'
+
+describe('calculate a moving average') {
+  it('works') {
+    start = Time.now
+    roller = Rollup.new(start, {})
+    roller.log(x: 60*7)
+    out = roller.roll(start+60)
+    out['300'][:x][:nps].should == 7
+    out['60'][:x][:nps].should == 7
+    roller.log(x: 60*7)
+    out = roller.roll(start+60*2)
+    out['300'][:x][:nps].should == 7
+    out['60'][:x][:nps].should == 7
+    roller.log(x: 60*1)
+    out = roller.roll(start+60*3)
+    out['300'][:x][:nps].should == ((60*7+60*7+60).to_f/(60*3)).round(4)
+    out['60'][:x][:nps].should == 1
+  }
+}

data/plugins/email/README.md

@@ -0,0 +1,30 @@
+# About
+
+The email plugin allows you to collect one or more metrics (matched via
+regexps) and relay them via mail on a configured interval.
+
+# Config
+
+```json
+{
+  "match" : ["^daemon_health", "^cpu"],
+  "interval" : 3600,  # 5MB / day inbox!
+  "unflatten" : true, # like compression, but ...
+
+  "to"   : "required",
+  "from" :  "you@host", # default ENV['USER'] + /etc/hostname
+  "smtp_settings" : {
+    # see Net::SMTP / ala ActionMailer
+    #   address, port, domain, # aka 'helo'
+    #   user_name, password, authentication # = plain|login|cram_md5
+  }
+}
+```
+
+# Regexp Matching
+
+The elements of the 'match' list are going to be interpolated
+into regular expressions, but quoted in JSON.  Thus, you will need to
+write "^cpu\\|user" to specifically match the 'cpu|user' metric.  (Hint:
+you may find the  "dot" any-character match e.g. "^cpu.user" to be more
+readable and unambiguous enough for most cases.)

data/plugins/email/email.json

@@ -0,0 +1,3 @@
+{
+
+}

data/plugins/email/email.rb

@@ -0,0 +1,52 @@
+
+require 'net/smtp'
+require 'mail'
+
+interval = config[:interval] || 60
+smtp_args = config[:smtp_settings]
+mail_args = {
+  to:      config[:to] || raise("must have 'to' address for e-mail"),
+  from:    config[:from] ||
+    (ENV['USER'] + '@' + File.new('/etc/hostname').readline.chomp),
+  subject: 'panoptimon update',
+}
+
+matches = config[:match] \
+  ? config[:match].map {|m| %r{#{m}}}
+  : nil
+
+current = {} # per-granule
+setup = ->() {
+EM.add_periodic_timer(interval, ->(){
+
+  body = current.keys.count == 0 ? "---\n"
+    : current.keys.sort {|a,b| a <=> b}.map {|t|
+    next if current[t].keys.count == 0
+    "#{t}: #{JSON.generate(current[t])}"}.join("\n") + "\n"
+  msg = ::Mail::Message.new( mail_args.merge({ body: body }) )
+  begin;
+  ::Net::SMTP.start( * smtp_args.values_at(
+    :address, :port, :domain,
+    :user_name, :password, :authentication
+  )) {|s| s.send_message(msg.to_s, msg.from[0], msg.to) }
+  rescue;
+    logger.warn("mail error: #{$!}")
+  end
+
+  current = {}
+})
+}
+
+->(metric) {
+  if setup; setup[] ; setup = nil; end
+
+  # collect / filter -> relay
+  t = Time.now.to_i
+  metric.each {|k,v|
+    next if matches and not(matches.find {|m| k =~ m})
+    store = current[t] ||= {}
+    store[k] = v # NOTE 1sec granularity / TODO conf[:granularity]?
+  }
+
+
+}

data/plugins/log_to_file/log_to_file.json

@@ -0,0 +1 @@
+{"file" : "/tmp/metrics.log"}

data/plugins/log_to_file/log_to_file.rb

@@ -0,0 +1,8 @@
+require "json"
+fh = File.new(config[:file] || raise("must have filename"), 'a')
+->(metric) {
+  # TODO handle failed writes / disk full?
+  fh.puts "#{Time.now.to_i} #{JSON.generate(metric)}"
+  # TODO flush on timer / only log once per second?
+  fh.flush
+}

data/plugins/log_to_logger/log_to_logger.json

@@ -0,0 +1,3 @@
+{"level" : "debug",
+"prefix" : "log_to_logger: ",
+"json"   : false}

data/plugins/log_to_logger/log_to_logger.rb

@@ -0,0 +1,7 @@
+level = config[:level] || 'debug'
+->(metric) {
+  monitor.logger.send(level) {
+    (config[:prefix] || '') +
+    (config[:json] ? JSON::generate(metric) : metric.inspect)
+  }
+}

data/plugins/metrics_http/README.md

@@ -0,0 +1,23 @@
+# Usage
+
+The '/metrics.json' request path will return a json object representing
+the current state of the metrics (TODO: maybe If-Modified-Since
+support.)
+
+To request only specific groups of metrics, append one or more metric
+names separated by '&'.  Metric names must match up to a '|' or the end.
+
+
+```none
+  $ GET 'http://localhost:8080/metrics.json'
+  { ... } # everything
+
+  $ GET 'http://localhost:8080/metrics.json/cpu'
+  {"cpu|user":"3","cpu|system":"2","cpu|idle":"95","cpu|wait":"0"}
+
+  $ GET 'http://localhost:8080/metrics.json/cpu|user'
+  {"cpu|user":"2"}
+
+  $ GET 'http://localhost:8080/metrics.json/iostat|sda|util&disk|/dev/sda3|space_free&disk|/dev/sda3|files_free'
+  {"disk|/dev/sda3|space_free":3.628937,"disk|/dev/sda3|files_free":336963,"iostat|sda|util":0.0}
+```

data/plugins/metrics_http/metrics_http.json

@@ -0,0 +1 @@
+{}

data/plugins/metrics_http/metrics_http.rb

@@ -0,0 +1,17 @@
+require "json"
+monitor.enable_cache
+monitor.http.mount('/metrics.json', ->(env) {
+  metrics = monitor.cached
+  match = env['PATH_INFO']
+  match = (match.length == 0 || match == '/') ? nil :
+    %r{^(?:#{
+      match.sub(/^\//, '').split(/&/).map {|it|
+        Regexp.escape(Rack::Utils.unescape(it))
+      }.join('|')
+    })(?:\||$)}
+  json = JSON.generate(
+    match ? metrics.select {|k,v| k =~ match} : metrics)
+  [200, {'Content-Type' => 'text/json'}, [json]]
+})
+
+return nil

data/plugins/riemann_stream/requires

@@ -0,0 +1 @@
+gem/riemann-client

data/plugins/riemann_stream/riemann_stream.json

@@ -0,0 +1,3 @@
+{
+  "host": "localhost",
+  "port": 5555 }

data/plugins/riemann_stream/riemann_stream.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'riemann/client'
+
+c = Riemann::Client.new(
+  host: config[:host] || 'localhost',
+  port: config[:port] || 5555,
+)
+
+hostname = config[:hostname] || Socket.gethostname
+
+->(metrics) {
+  c.send_maybe_recv(Riemann::Message.new(:events =>
+    metrics.keys.map {|k|
+      v = metrics[k]
+      v = v.respond_to?(:to_f) ? v.to_f : v ? 1 : 0
+      Riemann::Event.new(
+        host:    hostname,
+        service: k, metric:  v,
+        # TODO tags,description,ttl,state configurable?
+      )
+    }
+  ))
+}

data/plugins/status_http/requires

@@ -0,0 +1 @@
+gem/thin

data/plugins/status_http/status_http.json

@@ -0,0 +1 @@
+{}

data/plugins/status_http/status_http.rb

@@ -0,0 +1,60 @@
+require 'erb'
+require 'cgi'
+
+start = Time.now
+template = ERB.new <<EOT
+<html>
+  <head><title>Panoptimon</title></head>
+  <body>
+    <p>
+    <%= Time.now %> - running: <%= ((Time.now - start) / 60**2).round(2) %> hours
+    (<%= (%x{ps -o rss= -p #{Process.pid}}.to_f/1024).round(2) %> MB)
+    </p>
+
+    <h1>Config</h1>
+    <p><%= CGI.escapeHTML(JSON.pretty_generate(m.config.marshal_dump)).
+      gsub(/\n  /, '<br>&nbsp;&nbsp;').sub(/\n}/, '<br>}') %></p>
+
+    <h1>Plugins</h1>
+    <table style="padding:1">
+    <% m.loaded_plugins.each do |k,v| %>
+      <tr><td><%= CGI.escapeHTML(k) %></td>
+        <td><%= CGI.escapeHTML(JSON.generate(v)) %></td></tr></tr>
+    <% end %>
+    </table>
+
+    <h1>Collectors</h1>
+    <table style="padding:3">
+    <% m.collectors.each do |c| %>
+      <tr><td><%= CGI.escapeHTML(c.name) %></td>
+        <td><%= CGI.escapeHTML(JSON.generate(c.config)) %></td>
+        <td><%= c.last_run_time %></tr>
+    <% end %>
+    </table>
+
+    <h1>Current Metrics</h1>
+    <table border="1">
+    <thead><tr><th>name</th><th>value</th></thead>
+    <% m.cached.each do |k,v| %>
+      <tr><td><%= CGI.escapeHTML(k) %></td>
+        <td><%= CGI.escapeHTML(v.to_s) %></td></tr>
+    <% end %>
+    </table>
+  </body>
+</html>
+EOT
+render = ->(m) {
+  template.result(binding)
+}
+
+monitor.enable_cache
+monitor.http.match('/$', ->(env) {
+  monitor.logger.debug "passed me #{env}"
+  env['rack.logger'].debug "status page request for #{env['REMOTE_ADDR']}"
+  # NOTE ^- is the same as monitor.logger
+
+  [200, {'Content-Type' => 'text/html'}, [render.call(monitor)]]
+
+})
+
+return nil # no per-metric callback

data/sample_configs/1/collectors/alls_well.json

@@ -0,0 +1,6 @@
+{
+  "description": "json-only collector example",
+  "interval": 3.0,
+  "exec": "/bin/echo",
+  "args": ["-e", "{\"everything_ok\" : true}\\n\\c"]
+}

data/sample_configs/1/collectors/clock/clock

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require 'json'
+c = JSON.parse(ARGV[0])
+
+$stdout.sync = true
+
+$stderr.puts "interval: #{c['interval']}"
+while true
+  puts %Q{{"time" : #{Time.now.to_i}}}
+  sleep c['interval'] || 1
+end

data/sample_configs/1/collectors/clock.json

@@ -0,0 +1 @@
+{}

data/sample_configs/1/collectors/df/df.json

@@ -0,0 +1,6 @@
+{
+  "description": "df collector example",
+  "interval": 3.0,
+  "exec": "wrap_df",
+  "args": ["-kh"]
+}

data/sample_configs/1/collectors/df/wrap_df

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require 'json'
+cmd_args = ARGV[0]
+c = JSON.parse(ARGV[1])
+
+$stdout.sync = true
+cmd_output = %x{ df #{cmd_args} }
+
+lines = cmd_output.split(/\n/)
+
+headers = lines.shift()
+output  = {}
+
+lines.each do |line|
+  (fs, size, used, avail, use, mounted_on) = line.split 
+  h = { :fs => fs, :size => size, :used => used, :avail => avail, :percentage => use, :mounted_on => mounted_on } 
+  output[fs] = { :size => size, :used => used, :free => avail }
+end
+
+puts JSON.generate(:output => output)

data/sample_configs/1/collectors/load.json

@@ -0,0 +1 @@
+{}

data/sample_configs/1/panoptimon.json

@@ -0,0 +1,4 @@
+{
+  "collector_interval" : 360,
+  "collector_timeout": 10
+}

data/sample_configs/1/plugins/isup/isup.rb

@@ -0,0 +1,3 @@
+# TODO this should just hook into the EM loop and not do anything
+# with metrics?
+return nil

data/sample_configs/1/plugins/isup.json

@@ -0,0 +1 @@
+{}

data/sample_configs/1/plugins/log_to_file.json

@@ -0,0 +1 @@
+{"file" : "/tmp/metrics.log"}

data/sample_configs/err_handler/collectors/fail.json

@@ -0,0 +1,4 @@
+{
+  "description" : "failing collector",
+  "exec" : "/bin/false"
+}

data/sample_configs/err_handler/collectors/noisy.json

@@ -0,0 +1,5 @@
+{
+  "description" : "a noisy collector",
+  "exec" : "/usr/bin/perl",
+  "args" : ["-e", "$|=1; while() {sleep 1; warn qq/noises\\n/; print qq/{\"ok\" : true}\\n/}"]
+}

data/sample_configs/err_handler/collectors/noisy_failure.json

@@ -0,0 +1,5 @@
+{
+  "description" : "noisy failure",
+  "exec" : "/usr/bin/perl",
+  "args" : ["-e", "die q/cannot frobnate/"]
+}

data/sample_configs/err_handler/collectors/notfound.json

@@ -0,0 +1,4 @@
+{
+  "description" : "command does not exist",
+  "exec" : "/probablynosuchfileordirectoryinthislocationatleastwehopeso"
+}

data/sample_configs/err_handler/panoptimon.json

@@ -0,0 +1,3 @@
+{
+  "collector_interval" : 1
+}

data/sample_configs/passthru/collectors/beep.json

@@ -0,0 +1,5 @@
+{
+  "description" : "beeper",
+  "exec" : "/bin/echo",
+  "args" : ["-e", "{\"beep\" : 1}\\n\\c"]
+}

data/sample_configs/passthru/collectors/cat/collect_this.json

@@ -0,0 +1 @@
+{"x" : 1, "y" : 2, "z" : 3}

data/sample_configs/passthru/collectors/cat.json

@@ -0,0 +1,5 @@
+{
+  "description" : "static pass-thru collection for testing",
+  "exec" : "/usr/bin/perl",
+  "args" : ["-e", "print q{}.<>", "sample_configs/passthru/collectors/cat/collect_this.json"]
+}

data/sample_configs/passthru/panoptimon.json

@@ -0,0 +1,3 @@
+{
+  "collector_interval" : 1
+}

data/sample_configs/passthru/plugins/okcat/okcat.rb

@@ -0,0 +1,17 @@
+
+require 'json'
+expect = JSON.parse(File.new('sample_configs/passthru/collectors/cat/collect_this.json').read)
+expect.keys.each {|k| expect['cat|'+k] = expect.delete(k)}
+expect = JSON.generate(expect)
+return ->(m) {
+  return if m['beep|beep'] # TODO complain if it beeps twice
+  monitor.logger.debug expect
+  monitor.logger.debug { JSON.generate(m) }
+  if JSON.generate(m) == expect
+    puts 'ok'
+    monitor.stop
+  else
+    puts 'not ok'
+  end
+}
+

data/sample_configs/passthru/plugins/okcat.json

@@ -0,0 +1 @@
+{}

data/sample_configs/plugin_error/collectors/beep.json

@@ -0,0 +1,5 @@
+{
+  "description" : "beeper",
+  "exec" : "/bin/echo",
+  "args" : ["-e", "{\"beep\" : 1}\\n\\c"]
+}

data/sample_configs/plugin_error/panoptimon.json

@@ -0,0 +1 @@
+{"collector_interval" : 1}

data/sample_configs/plugin_error/plugins/error_always/error_always.rb

@@ -0,0 +1 @@
+return ->(metric) { raise 'error error trouble bubble' }

data/sample_configs/plugin_error/plugins/error_always.json

@@ -0,0 +1 @@
+{}

data/sample_configs/slow/collectors/slowbeep.json

@@ -0,0 +1,6 @@
+{
+  "description" : "too slow",
+  "timeout" : 2,
+  "exec" : "/usr/bin/perl",
+  "args" : ["-e", "$|=1; while() {sleep 3; print qq/{\"ok\" : true}\\n/}"]
+}

data/sample_configs/slow/panoptimon.json

@@ -0,0 +1 @@
+{}