panoptimon 0.0.2

data/lib/panoptimon/collector.rb

@@ -0,0 +1,135 @@
+# Copyright (C) 2012 Sourcefire, Inc.
+
+module Panoptimon
+
+require 'json';
+
+class Collector
+
+  include Panoptimon::Logger
+
+  attr_reader :name, :cmd, :config, :bus, :last_run_time, :interval
+  def initialize (args)
+    cmd = args.delete(:command) or raise "must have 'command' argument"
+    @cmd = cmd.class == Array ? cmd : Shellwords.shellsplit(cmd)
+    ->(exe) {
+      raise "no such file '#{exe}'" unless File.exist?(exe)
+      raise "command '#{exe}' is not executable" unless File.executable?(exe)
+    }.call(@cmd[0]) # TODO or maybe args[:interpreter]
+    @bus = args.delete(:bus) or raise "must have 'bus' argument"
+    args.each { |k,v| instance_variable_set("@#{k}", v) }
+
+    @name ||= 'unnamed'
+    @config ||= {}
+
+    @interval = config[:interval] ||= 60
+    @last_run_time = Time.at(-@interval)
+  end
+
+  def run
+    cmdc = @cmd + [JSON.generate(config)]
+
+    @last_run_time = Time.now # TODO .to_i ?
+
+    logger.info {"run command: #{cmdc}"}
+    @child = EM.popen3b(cmdc, CollectorSink, self)
+    @child.on_unbind { |status, errmess|
+      logger.error {"collector #{name} failed: #{status}" +
+        (errmess.nil? ? '' :
+          "\n  #{errmess.chomp.split(/\n/).join("\n  ")}")
+      } if(not(status.nil?) and status != 0)
+      @child = nil
+    }
+    logger.debug "timeout is: #{config[:timeout]}"
+    # XXX afaict, eventmachine just did not implement this:
+    # @child.set_comm_inactivity_timeout(config[:timeout])
+  end
+
+  def noise(mess)
+    logger.warn "collector/#{name} noise: #{mess.chomp}"
+  end
+
+  def running?
+    @child.nil? ? false : true
+  end
+
+end
+
+class Metric < Hash
+
+  def initialize (name, data)
+    name = data.delete('_name') if data['_name']
+    self.merge!(_flatten_hash({}, name, data))
+  end
+
+  def _flatten_hash (i,p,h)
+    h.each {|k,v|
+      ok = "#{p}|#{k}"
+      # TODO reject non-numeric data?
+      if k != '_info' and v.is_a?(Hash)
+        _flatten_hash(i, ok, v)
+      else
+        i[ok] = v
+      end
+    }
+    return i
+  end
+end
+
+module CollectorSink
+
+  def initialize (handler)
+    @handler = handler
+    @timeout = @handler.config[:timeout]
+    @interval = @handler.config[:interval]
+    timer_on
+  end
+
+  # reset / start timeout timer
+  def timer_on (opts={})
+    @timer.cancel unless @timer.nil?
+    length = @timeout + (opts[:with_interval] ? @interval : 0)
+    @timer = EventMachine::Timer.new(length) {
+      scrap = @buf ? " - #{@buf.flush}" : ''
+      @handler.logger.error "timeout on #{@handler.name}" + scrap
+      @handler.logger.debug {"pid #{get_pid}"}
+      close_connection()
+    }
+  end
+
+  def timer_off
+    @timer.cancel
+  end
+
+  def receive_data (data)
+    timer_on
+    @handler.logger.debug "incoming"
+    @buf ||= BufferedTokenizer.new("\n")
+    @buf.extract(data).each do |line|
+      timer_on(with_interval: true)
+      begin
+        data = JSON.parse(line)
+      rescue
+        # TODO feed errors up to the monitor
+        $stderr.puts "error parsing #{line.dump} - #{$!}"
+      end
+      @handler.logger.debug "line: #{line}"
+      @handler.bus.notify(Metric.new(@handler.name, data))
+    end
+  end
+
+  def receive_stderr (mess)
+    @handler.noise(mess)
+    (@err_mess ||= '') << mess
+  end
+
+  def on_unbind (&block); @on_unbind = block; end
+  def unbind
+    timer_off
+    @on_unbind.call(get_status.exitstatus, @err_mess)
+  end
+
+end
+
+end
+

data/lib/panoptimon/eventmonkeypatch/popen3.rb

@@ -0,0 +1,40 @@
+# Copyright (C) 2012 Sourcefire, Inc.
+
+# adapted from http://pastebin.com/C4hvAyKM
+# and https://gist.github.com/1333428
+
+# feed stderr into connection's receive_stderr()
+
+module EventMachine
+  class StderrHandler < EventMachine::Connection
+    def initialize(connection); @connection = connection; end
+    def receive_data(data); @connection.receive_stderr(data); end
+    def unbind; detach; end
+  end
+
+  def self.popen3b(cmd, handler=nil, *args)
+    klass = klass_from_handler(Connection, handler, *args)
+    raise "no command?" unless cmd.first
+    cmd.unshift(cmd.first) # -> execvp
+
+    original_stderr = $stderr.dup
+
+    begin
+      rd, wr     = IO.pipe
+
+      $stderr.reopen wr
+      s = invoke_popen(cmd)
+      $stderr.reopen original_stderr
+
+      connection = klass.new(s, *args)
+      EM.attach(rd, StderrHandler, connection)
+      @conns[s] = connection
+      yield(connection) if block_given?
+      connection
+    rescue
+      $stderr.reopen(original_stderr)
+      raise $!
+    end
+  end
+
+end

data/lib/panoptimon/http.rb

@@ -0,0 +1,63 @@
+# Copyright (C) 2012 Sourcefire, Inc.
+
+require 'thin'
+
+module Panoptimon
+class HTTP
+
+  include Logger
+
+  def initialize (args={})
+    @match = []
+    @mount = []
+    # TODO args[:config].http_port, ssl, etc
+    @http = Thin::Server.new('0.0.0.0', 8080, self);
+  end
+
+  def start
+    @http.backend.start
+  end
+
+  def call (env)
+    path = env['PATH_INFO']
+    return favicon(env) if path == '/favicon.ico'
+    # logger.debug { "#{path} => " + env.inspect }
+    if    go = @match.find {|x| path =~ x[0]}
+    elsif go = @mount.find {|x| path =~ %r{^#{x[0]}(/|$)}}
+      env['SCRIPT_NAME'] = go[0]
+      env['PATH_INFO'] = path.sub(%r{^#{go[0]}}, '')
+    else
+      return [404, {'Content-Type' => 'text/html'},
+        '<html><head><title>Not Found</title></head>' +
+        '<body><p>nope</p></body></html>']
+    end
+    env['rack.logger'] = logger
+    begin
+      return go[1].call(env)
+    rescue => ex
+      logger.error { "error: #{ex.message} #{ex.backtrace.join("\n  ")}" }
+      return [500, {'Content-Type' => 'text/html'}, ['bah']]
+    end
+  end
+
+  def favicon(env)
+    # TODO bundle / configure favicon?
+    # NOTE why doesn't rack/thin support .to_path per spec?
+    return [200, {'Content-Type' => 'image/x-icon'},
+      Pathname.new('/tmp/favicon.ico').open]
+  end
+
+  # regexp-match
+  def match (regexp, app)
+    regexp = %r{^#{regexp}} if regexp.class == String
+    @match.push([regexp, app])
+  end
+
+  # path prefix
+  def mount (point, app)
+    point.sub(%r{/*$}, '')
+    @mount.push([point, app])
+  end
+
+end
+end

data/lib/panoptimon/logger.rb

@@ -0,0 +1,19 @@
+# Copyright (C) 2012 Sourcefire, Inc.
+
+require "logger"
+
+module Panoptimon::Logger
+
+  def logger
+    Panoptimon::Logger.logger
+  end
+
+  def self.logger
+    @logger ||= Logger.new($stderr).tap {|l|
+      env_l = ENV.delete('LOG_LEVEL')
+      l.level = env_l.nil? ?
+        Logger::WARN : Logger.const_get(env_l.upcase)
+    }
+  end
+
+end

data/lib/panoptimon/monitor.rb

@@ -0,0 +1,154 @@
+# Copyright (C) 2012 Sourcefire, Inc.
+
+module Panoptimon
+class Monitor
+
+  include Panoptimon::Logger
+
+  attr_reader :config, :collectors, :plugins, :cached, :owd,
+    :loaded_plugins, :bus
+
+  def initialize (args={})
+    @collectors = []
+    @plugins    = {}
+    @loaded_plugins = {}
+    args.each { |k,v| instance_variable_set("@#{k}", v) }
+
+    @owd = Dir.pwd
+
+    me = self
+    @bus = EM.spawn { |metric| me.bus_driver(metric) }
+  end
+
+  def _dirjson (x)
+    x = Pathname.new(x)
+    x.entries.find_all {|f| f.to_s =~ /\.json$/i}.
+      map {|f| x + f}
+  end
+
+  def find_collectors; _dirjson(config.collectors_dir); end
+  def find_plugins;    _dirjson(config.plugins_dir);    end
+
+  def load_collectors
+    find_collectors.each {|f|
+      begin
+        _init_collector(_load_collector_config(f))
+      rescue => ex
+        logger.error "collector #{f} failed to load: \n" +
+          "  #{ex.message} \n  #{ex.backtrace[0]}"
+      end
+    }
+  end
+
+  def _load_collector_config (file)
+    conf = JSON.parse(file.read, {:symbolize_names => true})
+    base = file.basename.sub(/\.json$/, '').to_s
+    command = conf[:exec] ||= base
+    command = file.dirname + base + command unless command =~ /^\//
+    return conf.
+      merge({
+        name: base,
+        interval: (self.config.collector_interval || 99).to_i,
+        timeout:  (self.config.collector_timeout || 99).to_i
+      }) {|k,a,b| a}.
+      merge({command: command})
+  end
+
+  def _init_collector (conf)
+    name    = conf.delete(:name)
+    command = conf.delete(:command)
+    collector = Collector.new(
+      name: name,
+      bus: @bus,
+      command: [command.to_s] + conf[:args].to_a,
+      config: conf,
+    )
+    collectors << collector
+  end
+
+  def http
+    return @http unless @http.nil?
+    # TODO rescue LoadError => nicer error message
+    require 'panoptimon/http'
+    @http = HTTP.new
+  end
+
+  def empty_binding; binding; end
+  def load_plugins
+    find_plugins.each {|f| _init_plugin(_load_plugin_config(f)) }
+  end
+
+  def _load_plugin_config (file)
+    conf = JSON.parse(file.read, {:symbolize_names => true})
+    base = file.basename.sub(/\.json$/, '').to_s
+
+    # TODO support conf[:require] -> class.setup(conf) scheme?
+    rb = conf[:require] || "#{base}.rb"
+    rb = file.dirname + base + rb
+    return conf.
+      merge({
+        name: base,
+      }) {|k,a,b| a}.
+      merge({
+        base: base,
+        rb: rb
+      })
+  end
+
+  def _init_plugin (conf)
+    name = conf.delete(:name)
+    rb   = conf.delete(:rb)
+    setup = eval("->(name, config, monitor) {#{rb.open.read}\n}",
+      empty_binding, rb.to_s, 1)
+    callback = begin; setup.call(name, conf, self)
+      rescue; raise "error loading plugin '#{name}' - #{$!}"; end
+    logger.debug "plugin #{callback} - #{plugins[name]}"
+    plugins[name] = callback unless callback.nil?
+    loaded_plugins[name] = conf.clone # XXX need a plugin object?
+  end
+
+  def run
+
+    runall = ->() {
+      logger.debug "beep"
+      collectors.each {|c|
+        logger.info "#{c.cmd} (#{c.running? ? 'on ' : 'off'
+          }) last run time: #{c.last_run_time}"
+        next if c.last_run_time + c.interval > Time.now or c.running?
+        c.run
+      }
+    }
+    EM.next_tick(&runall)
+    logger.warn 'no collectors' if collectors.length == 0
+    minterval = collectors.map{|c| c.interval}.min
+    minterval = 67 if minterval.nil? # XXX should never happen
+    logger.debug "minimum: #{minterval}"
+    EM.add_periodic_timer(minterval, &runall);
+
+    @http.start if @http
+
+  end
+
+  def stop
+    EM.stop
+  end
+
+  def enable_cache(arg=true);
+    if arg; @cached ||= {}; else; @cached = nil; end
+  end
+
+  def bus_driver(metric)
+    logger.debug {"metric: #{metric.inspect}"}
+    metric.each {|k,v| @cached[k] = v} if @cached
+    plugins.each {|n,p|
+      begin p.call(metric)
+      rescue => e
+        logger.warn "plugin '#{n}' error: " +
+          "#{e}\n  #{e.backtrace[0].sub(%r{^.*/?(plugins/)}, '\1')}"
+        plugins.delete(n)
+      end
+    }
+  end
+
+end
+end

data/lib/panoptimon/util/string-with-as_number.rb

@@ -0,0 +1,5 @@
+class String; def as_number # from perlfaq4
+  self =~ %r{\A[+-]?(?=\.?\d)\d*\.?\d*(?:[Ee][+-]?\d+)?\z} \
+  ? (self =~ %r{[\.Ee]} ? self.to_f : self.to_i)
+  : nil
+end; end

data/lib/panoptimon/util.rb

@@ -0,0 +1,23 @@
+# Copyright (C) 2012 Sourcefire, Inc.
+
+module Panoptimon
+  module Util
+    VERSION = '0.0.1'
+
+    def self._os; @os ||= Gem::Platform.local.os.to_sym; end
+
+    # return osname
+    # or, given a hash, return the corresponding hash element and raise
+    # error if os not in hash keys
+    def self.os (dispatch={})
+      # TODO or mess with rbconfig + Config::CONFIG['host_os']
+      os = _os
+      return os unless dispatch.length > 0
+
+      it = dispatch[os] || dispatch[:default] or
+        raise "unsupported OS: #{os}"
+      return it.is_a?(Proc) ? it.call() : it
+    end
+
+  end
+end

data/lib/panoptimon/version.rb

@@ -0,0 +1,5 @@
+# Copyright (C) 2012 Sourcefire, Inc.
+
+module Panoptimon
+  VERSION = "0.0.2"
+end

data/lib/panoptimon.rb

@@ -0,0 +1,144 @@
+# Copyright (C) 2012 Sourcefire, Inc.
+
+require 'panoptimon/version'
+require 'panoptimon/logger'
+require 'panoptimon/monitor'
+require 'panoptimon/collector'
+
+require 'optparse'
+require 'ostruct'
+
+require 'rubygems'
+require 'eventmachine'
+require 'panoptimon/eventmonkeypatch/popen3.rb'
+
+module Panoptimon
+
+  class Panoptimon
+  end
+
+def self.load_options (args)
+  defaults = {
+    :daemonize      => true,
+    :config_dir     => '/etc/panoptimon/',
+    :config_file    => '%/panoptimon.json',
+    :collectors_dir => '%/collectors',
+    :plugins_dir    => '%/plugins',
+    :collector_interval => 60,
+    :collector_timeout  => 120,
+  }
+
+  options = ->() {
+    o = {}
+    OptionParser.new do |opts|
+
+      opts.on('-C', '--config-dir DIR',
+        "Config directory (#{defaults[:config_dir]})"
+      ) { |v| o[:config_dir] = v }
+
+      opts.on('-c', '--config-file FILENAME',
+        "Alternative configuration file ",
+        "(#{defaults[:config_file]})"
+      ) { |v| o[:config_file] = v.nil? ? '' : v }
+
+      opts.on('-D', '--[no-]foreground',
+        "Don't daemonize (#{not defaults[:daemonize]})"
+      ) { |v| o[:daemonize] = ! v }
+
+      ['collectors', 'plugins'].each { |x|
+        k = "#{x}_dir".to_sym
+        opts.on("--#{x}-dir DIR",
+          "#{x.capitalize} directory (#{defaults[k]})"
+        ) { |v| o[k] = v }
+      }
+
+      [:collectors, :plugins].each { |x|
+        opts.on('--list-'+x.to_s, "list all #{x} found"
+        ) { (o[:lists] ||= []).push(x) }
+      }
+
+      opts.on('-o', '--configure X=Y',
+        'Set configuration values'
+      ) { |x|
+        (k,v) = x.split(/=/, 2)
+        (o[:configure] ||= {})[k.to_sym] = v
+      }
+
+      opts.on('--show WHAT',
+        %q{Show/validate settings for:}, %q{  'config' / collector:foo / plugin:foo}
+      ) { |x| (k,v) = x.split(/:/, 2)
+        o[:show] = {k.to_sym => v||true}
+      }
+
+      opts.on('--plugin-test FILE',
+        'Load and test plugin(s).'
+      ) { |x| (o[:plugin_test] ||= []).push(x) }
+
+      opts.on('-d', '--debug', "Enable debugging."
+      ) { |v| o[:debug] = v }
+
+      opts.on('--verbose', "Enable verbose output"
+      ) { |v| o[:verbose] = v }
+
+      opts.on('-v', '--version', "Print version"
+      ) { 
+        puts "panoptimon version #{Panoptimon::VERSION}"
+        o[:quit] = true
+        opts.terminate
+      }
+
+      opts.on('--help-defaults', 'Show default config values'
+      ) {
+        puts JSON.pretty_generate(defaults)
+        o[:quit] = true
+        opts.terminate
+      }
+
+      opts.on("-h", "--help", "Show this message"
+      ) {
+        puts opts
+        o[:quit] = true
+        opts.terminate
+      }
+
+    end.parse!(args)
+
+    return o
+  }.call
+
+  return false if options[:quit]
+
+  render = ->(d, x) { # x with '%/' can be relative to dir d
+    f = "#{x}"; f.sub!(/^%\//, '').nil? ? f : File.join(d, f)
+  }
+
+  # default config file is relative to config dir
+  cfile = render.call(
+    options[:config_dir]  || defaults[:config_dir],
+    options[:config_file] || defaults[:config_file])
+
+  config = defaults.merge(
+    options[:config_file] == '' ? {} :
+      JSON.parse(File.read(cfile), {:symbolize_names => true})
+  ).merge(options);
+
+  config[:config_file] = cfile # for diagnostics
+
+  (config.delete(:configure) || {}).each { | k,v| config[k] = v }
+
+  [:collectors_dir, :plugins_dir].each { |d|
+    config[d] = render.call(config[:config_dir], config[d])
+  }
+
+  # make all paths absolute
+  [:config_file, :config_dir, :collectors_dir, :plugins_dir].each { |d|
+    config[d] = File.expand_path(config[d]) unless config[d] == ''
+  }
+
+  return OpenStruct.new(config).freeze
+
+end
+
+end
+
+# vim:ts=2:sw=2:et:sta

data/misc/collector_setup.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/ruby
+
+require 'panoptimon'
+
+count = 0
+duck = EM.spawn { |metric|
+  count += 1
+  puts "metric: #{metric.inspect} (#{count})"
+  EM.stop if count >= 500
+}
+
+c = Panoptimon::Collector.new(bus: duck,
+  command: 'sample_configs/1/collectors/clock/clock',
+  # command: %q{echo -e '{"everythings_ok" : 1}\n\c' },
+  config: {:interval => 0.5})
+
+puts "collector: #{c.inspect}"
+#c.logger.level = ::Logger::DEBUG
+
+EM.run {
+  c.run
+  EM.add_periodic_timer(1) { puts "running: #{c.running?}" }
+}

data/misc/monitor_setup.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/ruby
+
+require 'panoptimon'
+
+count = 0
+bus = EM.spawn { |metric|
+  count += 1
+  puts "metric: #{metric.inspect} (#{count})"
+  EM.stop if count >= 2000
+}
+
+m = Panoptimon::Monitor.new(
+  :collectors => [
+    Panoptimon::Collector.new(bus: bus,
+      command: 'sample_configs/1/collectors/clock/clock',
+      config: {:interval => 0.5}),
+    Panoptimon::Collector.new(bus: bus,
+      command: %q{echo -e '{"everythings_ok" : 1}\n\c' },
+      config: {:interval => 0.03})
+  ]
+)
+
+EM.run {
+  m.run
+}

data/misc/plugins_setup.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/ruby
+
+require 'panoptimon'
+
+m = Panoptimon::Monitor.new
+
+count = 0
+bus = EM.spawn { |metric|
+  count += 1
+  puts "metric: #{metric.inspect} (#{count})"
+  m.bus_driver(metric)
+  EM.stop if count >= 10
+}
+
+m.collectors << Panoptimon::Collector.new(
+  bus: bus,
+  command: 'sample_configs/1/collectors/clock/clock',
+  config: {:interval => 0.5},
+)
+
+m.plugins[:hello] = ->(metric){
+  puts "#{Time.now} - should do something with #{metric.inspect}"
+}
+
+EM.run { m.run }

data/misc/riemann-cli.rb

@@ -0,0 +1,33 @@
+#!/opt/ruby/bin/ruby
+#
+# Riemann data feeder
+# Feed json directly to a Riemann server
+#
+# Copyright Sourcefire, 2012
+# Benjamin Krueger <bkrueger@sourcefire.com>
+
+require 'rubygems'
+require 'json'
+require 'riemann/client'
+
+input_json = JSON[STDIN.read]
+
+puts "Service: #{input_json['service']}"
+puts "State: #{input_json['state']}"
+puts "Metric: #{input_json['metric']}"
+puts "Description: #{input_json['description']}"
+
+def submitEvent(event)
+  # New Riemann client
+  c = Riemann::Client.new host: 'riemann.example.com', port: 5555
+
+  # Send event to Riemann server
+  c << {
+    service: event['service'],
+    state: event['state'],
+    metric: event['metric'].to_i,
+    description: event['description']
+  }
+end
+
+submitEvent(input_json)