panoptimon 0.0.2

data/collectors/iostat/iostat

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+$stdout.sync = true # persistent process
+
+require 'json'
+opt = ARGV[0] ? JSON::parse(ARGV[0]) : {'interval' => 1, 'count' => 2}
+
+cmd = %w{iostat -xd}
+if opt['interval']
+  cmd.push(opt['interval'].to_s)
+  cmd.push(opt['count'].to_s) if opt['count']
+end
+p = IO.popen(cmd, 'r')
+
+[1,2].each { x = p.readline until x == "\n" } # header + first sample
+
+puts '{}' # beep
+
+def prep l
+  want = {
+    'kb_read/s'  => 'rkB/s',
+    'kb_write/s' => 'wkB/s',
+    'rrqm/s'     => 'rrqm/s',
+    'wrqm/s'     => 'wrqm/s',
+    'reads/s'    => 'r/s',
+    'writes/s'   => 'w/s',
+    'avgrq-sz'   => 'avgrq-sz',
+    'avgqu-sz'   => 'avgqu-sz',
+    'await'      => 'await',
+    'r_await'    => 'r_await',
+    'w_await'    => 'w_await',
+    'util'       => '%util',
+  }
+  head = l.chomp.split(/\s+/)
+  head = Hash[*head.zip(0..(head.length-1)).flatten]
+  raise "oops" unless head.values_at(*want.values).length == want.values.length
+  o = {}; want.each {|k,v| o[k] = head[v]}
+  return o
+end
+
+omap = nil
+group = {}
+
+until p.eof?
+  l = p.readline("\n\n").split(/\n/)
+  omap ||= prep(l[0])
+  puts JSON::generate(Hash[*l.drop(1).map {|x|
+    r = x.split(/\s+/)
+    [r[0], # device name
+      Hash[*omap.keys.map {|k| [k, r[omap[k]].to_f]}.flatten]]
+  }.flatten])
+
+end

data/collectors/iostat/iostat.json

@@ -0,0 +1 @@
+{"interval":1}

data/collectors/json/README.md

@@ -0,0 +1,27 @@
+# Config
+
+Each item in the 'collect' hash becomes the name of a collector output.
+The value can be simply a URL, or a hash ("url" entry required) with
+additional options.
+
+```json
+{
+  "collect" : {
+    "zebra" : "http://zebracage.localnet.zoo/status.json",
+    "monkey" : { "url" : "http://monkeycage.localnet.zoo/status.json" }
+  },
+  "default_timeout" : 3,
+
+}
+```
+
+# Output
+
+Each collector name is reported as a toplevel name, along with the
+collected hash -- as retrieved.  If there is a timeout (on connect or
+get), the value of the 'timeout' metric will be true.
+
+```json
+{ "_name" : "zebra", "temp" : 20, "rh" : 85, ... }
+{ "_name" : "monkey", "missing" : true, "last_seen" : 270 }
+```

data/collectors/json/json

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require 'net/http'
+
+ARGV[0] or raise "arguments required"
+conf = JSON.parse(ARGV[0], {symbolize_names: true})
+
+defaults = {
+  timeout: conf[:default_timeout] || 3,
+}
+
+raise "must have 'collect' value in config" unless conf[:collect]
+setup = conf[:collect].map {|k,v|
+  o = defaults.merge(v.is_a?(Hash) ? v : {url: v})
+  o[:url] = URI( o[:url].match(%r{^\w+://}) ?
+    o[:url] : 'http://' . o[:url] )
+  o[:name] ||= k
+  o
+}
+
+go = ->(url, o){
+  response = begin; Timeout::timeout(o[:timeout]) {
+    r = ::Net::HTTP.start(url.host, url.port,
+      :use_ssl      => url.scheme == 'https',
+    ).request(::Net::HTTP::Get.new(url.request_uri)) }
+    JSON.parse(r.body, {sybolize_names: true})
+  rescue Timeout::Error; {timeout: true}; end
+}
+
+# TODO consider persistence / staggering
+setup.each {|o|
+  output = go[o[:url], o]
+  output[:_name] = o[:name]
+  puts JSON.generate(output)
+}
+

data/collectors/json/json.json

@@ -0,0 +1 @@
+{"collect" : { "stuff" : "http://localhost:8089" }}

data/collectors/load/load

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'json'
+l = File.new('/proc/loadavg').read.split(/\s+/)
+(j,n) = l[3].split('/')
+
+# TODO support for systems with no /proc
+
+puts JSON::generate({
+  avg_1:  l[0].to_f,
+  avg_5:  l[1].to_f,
+  avg_15: l[2].to_f,
+  jobs:   j.to_i,
+  n:      n.to_i})
+

data/collectors/load/load.json

@@ -0,0 +1 @@
+{}

data/collectors/memcached/memcached

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+# Copyright (C) 2012 Sourcefire, Inc.
+
+
+require 'json'
+opt = ARGV[0] ? JSON::parse(ARGV[0], {symbolize_names: true}) : {}
+
+# TODO require 'file/which' ne see stackoverflow 2108727?
+require 'pathname'
+class Pathname; def self.which(cmd)
+  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+  ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+    path = Pathname.new(path)
+    exts.each { |ext|
+      it = path + "#{cmd}#{ext}"
+      return it if it.executable?
+    }
+  end
+  raise "No such command '#{cmd}' in $PATH"
+end; end
+
+class String; def as_number # from perlfaq4
+  self =~ %r{\A[+-]?(?=\.?\d)\d*\.?\d*(?:[Ee][+-]?\d+)?\z} \
+  ? (self =~ %r{[\.Ee]} ? self.to_f : self.to_i)
+  : nil
+end; end
+
+class Array; def to_h ; Hash[*self.flatten]; end; end
+
+hosts = opt[:hosts] || {default: 'localhost'}
+hosts.values.each {|x| x += ':11211' unless x =~ /:\d+$/}
+hostmap = hosts.map {|k,v| [v,k]}.to_h
+
+runcmd = [opt[:memstat] || Pathname.which('memstat')].flatten.push(
+  '--servers=' + hostmap.keys.join(',')
+)
+
+lines = IO.popen(runcmd).readlines.map {|l| l.chomp.split(/:\s+/, 2)}
+puts JSON.generate(lines.inject([]) {|l,p|
+  if p[0] == 'Server'
+    p[1] =~ %r{(.*)\s\((\d+)\)$} or raise "'#{p[1]}' - unexpected format"
+    host,port = $1,$2
+    name = hostmap["#{host}:#{port}"] or raise "unnamed host:port '#{host}:#{port}'"
+    l.push([name, {_info: {host: host, port: port.to_i}}])
+  else
+    key = p[0].sub(/^\s+/, '')
+    m = p[1].as_number
+    if m.nil?
+      l[-1][-1][:_info][key] = p[1]
+    else
+      l[-1][-1][key] = m
+    end
+  end
+  l
+}.to_h)

data/collectors/memcached/memcached.json

@@ -0,0 +1,7 @@
+{
+  "hosts": {
+    "jim" : "10.0.0.1:11211",
+    "bob" : "10.0.0.2:11211"
+  },
+  "memstat" : "/usr/bin/memstat"
+}

data/collectors/memcached/test-notes.txt

@@ -0,0 +1,3 @@
+ # just lie about where memstat is to replay a static copy
+ ruby collectors/memcached/memcached '{"memstat" : ["cat", "/tmp/memstat-sample.txt"]}'
+

data/collectors/memory/memory

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'json'
+
+config = ARGV[0] ? JSON::parse(ARGV[0]) : {}
+
+$stdout.sync = true
+
+def GB b; (b.to_f / 1024**2).round(6); end # GB significant down to 4kB
+
+while(true) do
+
+  info = File.readlines('/proc/meminfo').map {|l|
+    l = l.chomp.split(/:\s+/, 2)
+    raise "units assumption failed" if
+      l[1].sub!(/\s+(kB)$/, '') and not($+ == 'kB')
+    l
+  }
+  i = Hash[*info.flatten]
+
+  out = {
+    total:     GB(i['MemTotal']),
+    free:      GB(i['MemFree']),
+    buffers:   GB(i['Buffers']),
+    cached:    GB(i['Cached']),
+    swapped:   GB(i['SwapTotal'].to_i - i['SwapFree'].to_i),
+    swap_free: GB(i['SwapFree']),
+  }
+  puts JSON::generate(out)
+
+  break unless config.include?('interval')
+  sleep config['interval']
+end

data/collectors/memory/memory.json

@@ -0,0 +1 @@
+{"interval":1}

data/collectors/mysql_status/mysql_status

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'mysql';
+require 'json';
+
+class String; def as_number # from perlfaq4
+  self =~ %r{\A[+-]?(?=\.?\d)\d*\.?\d*(?:[Ee][+-]?\d+)?\z} \
+  ? (self =~ %r{[\.Ee]} ? self.to_f : self.to_i)
+  : nil
+end; end
+
+class Mysql_status
+
+def self.get_status (db)
+  rs = db.query("SHOW STATUS")
+  h = {}
+  while( r = rs.fetch_row )
+    h[r[0].downcase] = r[1]
+  end
+  return h
+end
+
+def self.get_slave_status (db)
+  rs = db.query('SHOW SLAVE STATUS')
+  h = rs.fetch_hash or return {}
+  raise "too many results" if rs.num_rows > 1
+  return Hash[* h.keys.map {|k| [k.downcase, h[k]]}.flatten]
+end
+
+def self.run (config)
+  db = Mysql.new(*
+    %w{hostname username password database port socket}.
+    map {|k| config[k]})
+
+  m = get_status(db)
+  m.merge(get_slave_status(db))
+  info = {}
+  m.keys.each {|k|
+    n = m[k].as_number
+    if n
+      m[k] = n 
+    else
+      info[k] = m.delete(k)
+    end
+  }
+  m[:_info] = info if info.length > 0
+  puts JSON::generate(m)
+end
+
+end
+
+Mysql_status.run(JSON::parse(ARGV[0])) if __FILE__ == $0

data/collectors/mysql_status/mysql_status.json

@@ -0,0 +1,4 @@
+{
+  "description" : "mysql status and slave status collector.  Config must provide some valid combination of hostname,username,password,database,port,socket to connect.",
+  "socket" : "/tmp/mysql.socket"
+}

data/collectors/network/network

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+
+$stdout.sync = true # persistent process
+
+require 'json'
+opt = ARGV[0] ? JSON::parse(ARGV[0], {symbolize_names: true})
+  : {interval: 1, once: 1}
+opt[:interval] ||= 5
+
+cmd = %w{netstat -an --protocol=inet}
+
+opt[:report] or raise "'report' section must be configured"
+
+# TODO read /etc/services and support default numeric port match?
+
+# allow addr to be regexp, glob, or literal
+def _escape (addr)
+  addr =~ /\\./ ? addr
+  : addr =~ /\.\*/ ? addr.gsub(/\*/, '\d+').gsub(/\./, '\.')
+  : Regexp.escape(addr)
+end
+
+# build a report matchlist
+report = opt[:report].keys.map { |k|
+  c = opt[:report][k]
+  {
+    name: k,
+    match: %r{^\w+\s+\d+\s+\d+\s+ # proto,recv,send
+      #{c[:localaddr] ? _escape(c[:localaddr]) : '\d+\.\d+\.\d+.\d+'}
+      :
+      #{c[:localport] ? c[:localport].to_s : '(?:\d+|\*)'}
+      \s+
+      #{c[:remoteaddr] ? _escape(c[:remoteaddr]) : '\d+\.\d+\.\d+.\d+'}
+      :
+      #{c[:remoteport] ? c[:remoteport].to_s : '(?:\d+|\*)'}
+      \s+
+      #{c[:state] ? c[:state].upcase : '\w+'}
+      \s*$
+    }x,
+  }
+}
+
+
+# TODO support netstat --continous if interval < 2 or something?  Would
+# require output to be delayed or something since there is no end marker
+# to the netstat output.
+
+while true
+
+  p = IO.popen(cmd)
+  lines = p.readlines.map {|l| l.chomp}
+  p.close # autoclose my foot
+  lines.shift; lines.shift
+
+  out = {}
+  lines.each { |l|
+    report.find_all {|r| l =~ r[:match]}.each {|match|
+      out[match[:name]] ||= 0
+      out[match[:name]] += 1
+    }
+  }
+
+  puts JSON::generate(out)
+
+  break if opt[:once]
+  sleep opt[:interval]
+end

data/collectors/network/network.json

@@ -0,0 +1,18 @@
+{
+  "report": {
+    "gemserver": {
+      "localaddr": "127.0.0.1",
+      "localport": 8808
+    },
+    "ssh" : {
+      "localport" : 22,
+      "state" : "ESTABLISHED"
+    },
+    "daemons" : { "state" : "LISTEN" },
+    "servers" : { "state" : "LISTEN", "localaddr" : "0.0.0.0" },
+    "self" : {"state" : "ESTABLISHED", "remoteaddr" : "127.0.0.1"},
+    "east" : {"remoteaddr" : "66\\.225\\.\\d+\\.\\d+"},
+    "west" : {"remoteaddr" : "173.194.*.*"}
+  }
+  ,"interval" : 1
+}

data/collectors/nginx/README.md

@@ -0,0 +1,32 @@
+#Description
+
+The Panoptimon collector 'nginx' is intended to retrieve statistics from a
+running nginx instance. The nginx module 'HTTPStubStatus' is required (see
+below.)
+
+# Config
+
+```json
+ {"url": "http://localhost/nginx-status"}
+```
+
+# Output
+
+```
+{"requests":"137","writing":"1","total":"1","reading":"0"}
+```
+
+
+# Requisite Server Setup
+
+The nginx server must provide the information via its HTTPStubStatus
+module. Within a server definition simply add the following.
+
+```
+location /nginx-status {
+	stub_status on;
+	access_log   off;
+	allow 127.0.0.1;
+	deny all;
+}
+```

data/collectors/nginx/lib/panoptimon-collector-nginx/nginx.rb

@@ -0,0 +1,45 @@
+module Panoptimon
+  module Collector
+    class Nginx
+
+      attr_reader :uri
+      def initialize(options={})
+        options = default_options.merge!(options)
+        @uri = URI(options[:url])
+      end
+
+      def default_options
+        {url: 'http://localhost/nginx-status'}
+      end
+
+      def connect
+        @connect ||= Net::HTTP.new(uri.host, uri.port)
+      end
+
+      def request
+        @request ||= Net::HTTP::Get.new(uri.request_uri)
+      end
+
+      def response
+        @response ||= connect.request(request)
+      end
+
+      def info(body=nil)
+        body ||= response.body
+        raise "not a status report page" if body.match('<') and
+          not body.match(/^Active/)
+        begin
+          {
+            requests: body.match('\s+(\d+)\s+(\d+)\s+(\d+)')[3].to_i,
+            total:    body.match('connections:\s(\d+)')[1].to_i,
+            reading:  body.match('Reading:\s(\d+)')[1].to_i,
+            writing:  body.match('Writing:\s(\d+)')[1].to_i
+          }
+        rescue NoMethodError
+          raise "probably not status page"
+        end
+      end
+
+    end
+  end
+end

data/collectors/nginx/lib/panoptimon-collector-nginx.rb

@@ -0,0 +1,2 @@
+require 'net/http'
+require 'panoptimon-collector-nginx/nginx'

data/collectors/nginx/nginx

@@ -0,0 +1,11 @@
+#! /usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), 'lib')))
+require 'rubygems'
+require 'json'
+require 'panoptimon-collector-nginx'
+
+client = Panoptimon::Collector::Nginx.new(
+  ARGV[0] ? JSON.parse(ARGV[0], {symbolize_names: true}) : {})
+puts client.info.to_json
+

data/collectors/nginx/nginx.json

@@ -0,0 +1,3 @@
+{
+  "url": "http://localhost/nginx-status"
+}

data/collectors/nginx/panoptimon-collector-nginx.gemspec

@@ -0,0 +1,4 @@
+Gem::Specification.new { |s|
+
+  s.add_dependency 'net-http'
+}

data/collectors/ping/README.md

@@ -0,0 +1,54 @@
+
+# Permissions
+
+Your ping binary must be `setcap cap_net_raw=ep` or setuid root.  The
+collector itself requires no special permissions.
+
+# Config
+
+```json
+  {
+    "hosts" : [
+      "example.com",
+      # or with per-host settings
+      {"host" : "localhost", "timeout" : 1, "count" : 1}
+    ],
+
+    # defaults
+    "default_timeout" : 4,
+    "default_count"   : 2,
+    "ping"            : "/bin/ping", # defaults to search $PATH
+
+    # standard collector configuration
+    "interval" : ...
+    "timeout" : ...
+    "exec" : "ping/ping",
+    "args" : [],
+  }
+```
+
+# Output
+
+{
+   "ping|example.com|loss"     : 0,    # 0-1 float
+   "ping|example.com|rtt_avg"  : 54.47,
+   "ping|example.com|rtt_max"  : 55.914,
+   "ping|example.com|rtt_min"  : 53.027,
+   "ping|example.com|rtt_mdev" : 1.462,
+   "ping|example.com|rx"       : 8, # received
+   "ping|example.com|tx"       : 8, # transmitted
+   "ping|example.com|_info"    : { "resolved" : "192.0.43.10" }
+}
+
+
+# Errors
+
+{
+   "ping|whatever.example.com|error" : 2, # return code
+   "ping|whatever.example.com|_info" : {
+      "error" : "ping: unknown host whatever.example.com"
+   },
+   "ping|whatever.example.com|loss" : 1,
+   "ping|whatever.example.com|rx"   : 0,
+   "ping|whatever.example.com|tx"   : 0
+}

data/collectors/ping/ping

@@ -0,0 +1,57 @@
+#! /usr/bin/env ruby
+
+require 'json'
+     
+conf = ARGV[0] ? JSON.parse(ARGV[0], {symbolize_names: true}) : {}
+
+bin_ping = [conf[:ping] || ['ping']].flatten
+
+defaults = {
+  timeout: conf[:default_timeout] || 4,
+  count:   conf[:default_count] || 2,
+}
+
+setup = Hash[conf[:hosts].map {|h|
+  (host, opt) = h.is_a?(Hash) ? [h[:host], h.tap{|x| x.delete(:host)}]
+    : [h, {}]
+  opt = defaults.merge(opt)
+  [host, opt]
+}]
+
+pings = Hash[setup.map {|host, opt|
+  args = ['-c', opt[:count], '-W', opt[:timeout]]
+  p = IO.popen((bin_ping + args + [host]).map {|e| e.to_s} +
+    [:err => [:child, :out]])
+  text = p.readlines.join('')
+  p.close
+  if not($?.success?) and not(text.match(%r{% packet loss}))
+    raise "runnaway ping" unless $?.exited? # never
+    next [host, { error: $?.exitstatus, tx: 0, rx: 0, loss: 1.0,
+      _info: {error: text.chomp}}]
+  end
+
+  matched = text.match(%r{
+    ^PING\ [^(]+\((?<resolved>[^)]+)\).*
+    ^(?<tx>\d+)\ packets\ transmitted,\s+
+     (?<rx>\d+)\ received,\s+
+     (?<loss>\d+)%\ packet\ loss,
+  }xm) or raise "sad bunny said #{text} - cannot parse"
+  result = Hash[ %w{tx rx loss}.map {|n| [n, matched[n].to_i]} ].
+    merge({ _info: { resolved: matched[:resolved], } })
+  result['loss'] /= 100.0
+
+  if matched = text.match(%r{
+      ^rtt\ min/avg/max/mdev\ =\s+
+      (?<rtt_min>\d*\.\d+)/
+      (?<rtt_avg>\d*\.\d+)/
+      (?<rtt_max>\d*\.\d+)/
+      (?<rtt_mdev>\d*\.\d+)\ ms
+    }xm)
+    result.merge!(Hash[ %w{rtt_min rtt_avg rtt_max rtt_mdev}.map {|n|
+      [n, matched[n].to_f]} ])
+  end
+  
+  [host, result]
+}]
+
+puts JSON.generate(pings)

data/collectors/ping/ping.json

@@ -0,0 +1,7 @@
+{
+  "hosts": [
+    "example.com",
+    {"host" : "localhost", "timeout" : 1, "count" : 1}
+  ]
+  ,"default_count" : 8
+}

data/collectors/process/README.md

@@ -0,0 +1,36 @@
+Returns information about a process, or set of processes.
+
+Can look simply for a process name, or match against an expression, search for all processed owned by a user... can also match processes that match certain characteristics as defined in the optional arguments section.
+
+  ps -wwo 'pid,time,etime,thcount,pcpu,ni,pri,vsz,rss,command' \
+    -p $(pgrep -d, sshd)
+
+# Config
+
+The "checks" hash contains a list of check names, plus info for invoking pgrep.
+
+```json
+{
+  interval: 60,
+  checks: {
+    sshd: {
+      pattern: "^sshd", # also 'full:  "^/usr/sbin/sshd"
+      user: "root,daemon"
+    }
+  }
+}
+```
+
+# Return:
+
+Information about each matched process, plus a count.
+
+  process|ssh|count => 1
+  process|ssh|0|time => 400
+  process|ssh|0|etime => 6000
+  process|ssh|0|pcpu => 0.2
+  process|ssh|0|thcount => 1
+  process|ssh|0|rss => ...
+  process|ssh|0|nice => ...
+  process|ssh|0|priority => ...
+  process|ssh|_info => { 'cmd' => [...] }