ganymed 0.1.0

data/lib/ganymed/event.rb

@@ -0,0 +1,102 @@
+module Ganymed
+
+  ##
+  # An incoming event is a hash-like object and contains the following keys:
+  #
+  # n::  namespace (e.g. +os.cpu.idle+, +nginx.requests+, etc)
+  # m::  event modifiers (e.g. +realtime+)
+  # c::  a consolidation function used to build this event from multiple
+  #      samples
+  # o::  origin (e.g. +web1.example.com+, +myapplication+)
+  # t::  timestamp as seconds since epoch
+  # v::  the value in integer or floating-point representation
+  #
+  class Event
+    # The namespace of this event.
+    # @return [String]
+    attr_accessor :ns
+
+    # The event modifiers.
+    # @return [Array]
+    attr_accessor :modifiers
+
+    # The consolidation function used to build this event from multiple
+    # samples.
+    # @return [String]
+    attr_accessor :cf
+
+    # The origin of this event.
+    # @return [String]
+    attr_accessor :origin
+
+    # The timestamp of this event.
+    # @return [Fixnum]
+    attr_accessor :timestamp
+
+    # The actual value of this sample.
+    # @return [Fixnum, Float]
+    attr_accessor :value
+
+    # Create a new {Event} instance from +obj+.
+    #
+    # @return [Event] The parsed event instance.
+    def self.parse(obj)
+      new.tap do |event|
+        event.ns = obj['n']
+        event.modifiers = obj['m']
+        event.cf = obj['c']
+        event.origin = obj['o']
+        event.timestamp = obj['t'].to_i
+
+        value = obj['v']
+        if value.is_a?(Float) and (value.nan? or value.infinite?)
+          event.value = nil
+        else
+          event.value = value
+        end
+
+        event.validate
+      end
+    end
+
+    # Convert this event into a hash with long keys. This hash cannot be parsed
+    # by #parse which accepts short keys for effiency.
+    #
+    # @return [Hash]
+    def to_h
+      {
+        :ns => ns,
+        :cf => cf,
+        :origin => origin,
+        :timestamp => timestamp,
+        :value => value
+      }
+    end
+
+    # Check if this event is a realtime event.
+    #
+    # @return [TrueClass,FalseClass]
+    def realtime?
+      @modifiers.include?("realtime")
+    end
+
+    # Validate this event.
+    def validate
+      unless @ns =~ /^[a-z][a-z0-9_\.-]+[a-z0-9]$/
+        raise "invalid namespace: #{@ns.inspect}"
+      end
+
+      if @ns =~ /^(system|ganymed)\./
+        raise "namespace is reserved: #{@ns.inspect}"
+      end
+
+      if @timestamp < 0
+        raise "invalid timestamp: #{@timestamp.inspect}"
+      end
+
+      unless @value.is_a?(Fixnum) or @value.is_a?(Float)
+        raise "invalid value in #{@ns}: #{@value.inspect} (#{@value.class.name})"
+      end
+    end
+  end
+end

data/lib/ganymed/master.rb

@@ -0,0 +1,150 @@
+require 'eventmachine'
+require 'madvertise/ext/config'
+require 'madvertise/ext/environment'
+require 'mixlib/cli'
+require 'servolux'
+require 'socket'
+
+require 'ganymed'
+require 'ganymed/collector'
+require 'ganymed/processor'
+require 'ganymed/sampler'
+require 'ganymed/version'
+
+module Ganymed
+  # @private
+  class Master < ::Servolux::Server
+    include Configuration::Helpers
+
+    def initialize(cli)
+      $0 = "ganymed"
+
+      log.level = :debug if cli[:debug]
+
+      @default_config_file = File.join(LIB_DIR, 'ganymed/config.yml')
+      @config_file = cli[:config_file]
+
+      @pid_file = config.pid_file.tap{}
+
+      super('ganymed', :interval => 1, :logger => log, :pid_file => @pid_file)
+    end
+
+    def run
+      unless EventMachine.reactor_running?
+        @reactor.join if @reactor.is_a?(Thread)
+        log.info("starting Ganymed reactor in #{Env.mode} mode")
+        @reactor = Thread.new do
+          begin
+            EventMachine.run { setup_reactor }
+          rescue Exception => exc
+            log.exception(exc)
+          end
+        end
+      end
+
+      if config.profiling.gcprofiler
+        puts GC::Profiler.report
+      end
+    end
+
+    def setup_reactor
+      Processor.new(config) if config.processor.enabled
+      Sampler.new(config) if config.sampler.enabled
+      Collector.new(config) if config.collector.enabled
+    end
+
+    def before_starting
+      if config.profiling.perftools
+        log.info("activating PerfTools CPU profiler")
+        require 'perftools'
+        PerfTools::CpuProfiler.start('profile')
+      end
+
+      if config.profiling.gcprofiler
+        log.info("activating GC::Profiler")
+        GC::Profiler.enable
+      end
+
+      if config.profiling.rubyprof
+        require 'ruby-prof'
+        log.info("activating RubyProf")
+        RubyProf.start
+      end
+    end
+
+
+    def after_stopping
+      log.info("shutting down Ganymed reactor")
+      EventMachine.stop_event_loop if EventMachine.reactor_running?
+      @reactor.join if @reactor.is_a?(Thread)
+
+      if config.profiling.perftools
+        PerfTools::CpuProfiler.stop
+      end
+
+      if config.profiling.rubyprof
+        result = RubyProf.stop
+        result.eliminate_methods!([
+          /Queue#pop/,
+          /Queue#push/,
+          /Mutex#synchronize/,
+          /Mutex#sleep/,
+        ])
+        printer = RubyProf::MultiPrinter.new(result)
+        printer.print(:path => "prof", :profile => "ganymed")
+      end
+
+      log.info("done shutting down. exiting ...")
+    end
+
+    class CLI
+      include Mixlib::CLI
+
+      option :config_file,
+        :short => '-c CONFIG',
+        :long => '--config CONFIG',
+        :description => "The configuration file to use"
+
+      option :debug,
+        :short => '-D',
+        :long => '--debug',
+        :description => "Enable debug output",
+        :boolean => true
+
+      option :daemonize,
+        :short => '-d',
+        :long => '--daemonize',
+        :description => "Daemonize the server process",
+        :boolean => true
+
+      option :kill,
+        :short => '-k',
+        :long => '--kill',
+        :description => "Kill the currently running daemon instance",
+        :boolean => true
+
+      option :environment,
+        :short => '-e ENVIRONMENT',
+        :long => '--environment ENVIRONMENT',
+        :description => "Set the daemon environment",
+        :default => "development",
+        :proc => Proc.new { |value| ENV[Env.key] = value }
+
+      option :help,
+        :short => '-h',
+        :long => '--help',
+        :description => "Show this message",
+        :on => :tail,
+        :boolean => true,
+        :show_options => true,
+        :exit => 0
+
+      def self.parse_options
+        Env.key = 'GANYMED_ENV'
+        new.tap do |cli|
+          cli.parse_options
+        end
+      end
+    end
+  end
+end

data/lib/ganymed/mongodb.rb

@@ -0,0 +1,39 @@
+require 'mongo'
+
+module Ganymed
+  # @private
+  class MongoDB
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+      log.info("using MongoDB at #{config.host}:#{config.port}/#{config.database}")
+    end
+
+    def connection
+      @connection ||= ::Mongo::Connection.new(config.host,
+                                              config.port,
+                                              :pool_size => 5,
+                                              :pool_timeout => 5)
+    end
+
+    def db
+      @db ||= connection.db(config.database)
+    end
+
+    def collection(ns)
+      db[ns].tap do |col|
+        col.ensure_index([['c', ::Mongo::ASCENDING]])
+        col.ensure_index([['o', ::Mongo::ASCENDING]])
+        col.ensure_index([['t', ::Mongo::ASCENDING]])
+        col.ensure_index([['c', ::Mongo::ASCENDING], ['o', ::Mongo::ASCENDING]])
+        col.ensure_index([['o', ::Mongo::ASCENDING], ['t', ::Mongo::ASCENDING]])
+        col.ensure_index([['c', ::Mongo::ASCENDING], ['o', ::Mongo::ASCENDING], ['t', ::Mongo::ASCENDING]], :unique => true)
+      end
+    end
+
+    def method_missing(method, *args, &block)
+      db.__send__(method, *args, &block)
+    end
+  end
+end

data/lib/ganymed/processor.rb

@@ -0,0 +1,93 @@
+require 'eventmachine'
+require 'msgpack'
+
+require 'ganymed/event'
+require 'ganymed/mongodb'
+require 'ganymed/websocket'
+
+module Ganymed
+  class Processor
+    def initialize(config)
+      @config = config
+      @db = MongoDB.new(config.processor.mongodb)
+      @websocket = Websocket.new(config.processor.websocket, @db)
+      listen!
+    end
+
+    def listen!
+      @config.processor.listen.tap do |listen|
+        log.info("processing metrics on udp##{listen.host}:#{listen.port}")
+        EM.open_datagram_socket(listen.host, listen.port, Connection) do |connection|
+          connection.db = @db
+          connection.websocket = @websocket
+        end
+      end
+    end
+
+    module Connection
+      attr_accessor :db, :websocket
+
+      def receive_data(data)
+        EM.defer do
+          begin
+            data = MessagePack.unpack(data)
+            case data['_type'].to_sym
+            when :event
+              process_event(data)
+            when :metadata
+              process_metadata(data)
+            end
+          rescue Exception => exc
+            log.exception(exc)
+          end
+        end
+      end
+
+      def process_event(data)
+        event = Event.parse(data)
+
+        # insert origin + namespace into metadata
+        metadata = db['metadata'].find_and_modify({
+          query: {:_id => event.origin},
+          update: {:$addToSet => {:namespaces => event.ns}},
+          upsert: true,
+        })
+
+        # set defaults in case find_and_modify returned an empty document
+        metadata['_id'] ||= event.origin
+        metadata['namespaces'] ||= []
+
+        # if metadata has changed notify websocket clients
+        if not metadata['namespaces'].include?(event.ns)
+          metadata['namespaces'] |= [event.ns]
+          websocket.send(:metadata, [metadata])
+        end
+
+        # send the event to websocket clients
+        websocket.each do |connection|
+          connection.publish(event)
+        end
+
+        # do not store realtime events into mongodb
+        return if event.realtime?
+
+        # insert event into ns collection
+        db.collection(event.ns).update({
+          :c => event.cf,
+          :o => event.origin,
+          :t => event.timestamp
+        }, {
+          :$set => {:v => event.value}
+        }, :upsert => true)
+      end
+
+      def process_metadata(data)
+        db['metadata'].update({
+          :_id => data['origin'],
+        }, {
+          :$set => {:data => data['data']}
+        }, :upsert => true)
+      end
+    end
+  end
+end

data/lib/ganymed/sampler/counter.rb

@@ -0,0 +1,32 @@
+require 'ganymed/sampler/datasource'
+
+module Ganymed
+  class Sampler
+
+    ##
+    # A Counter {DataSource} processes samples counting a specific event (such
+    # as a webserver request, user login, etc) and produces a rate/s gauge.
+    #
+    class Counter < DataSource
+      def flush(tick, &block)
+        each(tick) do |ns, origin, events|
+          values = events.sort_by do |event| # sort by timestamp
+            event[0]
+          end.group_by do |event| # group by timestamp
+            event[0].to_i
+          end.map do |_, events| # calculate counts for each second.
+            events.reduce(0) do |memo, event| # sum(event.count)
+              memo + event[1]
+            end
+          end
+
+          yield ns, origin, values
+        end
+      end
+
+      def feed(ns, origin, ts, value)
+        add(ns, origin, [ts, value])
+      end
+    end
+  end
+end

data/lib/ganymed/sampler/datasource.rb

@@ -0,0 +1,92 @@
+module Ganymed
+  class Sampler
+
+    ##
+    # A DataSource processes samples from various sources.  Samples are stored
+    # to an in-memory buffer and flushed to the {Processor} once for every
+    # tick.
+    #
+    # All DataSources interpolate their samples to gauge style values before
+    # flushing to the processor, e.g. a counter (like requests, or bytes sent
+    # over the network) is interpolated to a rate/s gauge (e.g. requests/s or
+    # bytes/s).
+    #
+    class DataSource
+
+      attr_reader :ticks, :buffer
+
+      # Create a new DataSource with buffers for every +tick+ in +ticks+.
+      #
+      # @param [Array] ticks  All ticks that should be stored in the buffer.
+      #
+      def initialize(ticks)
+        @ticks = ticks
+        @mutex = Mutex.new
+        @buffer = Hash.new do |buffer, tick|
+          buffer[tick] = Hash.new do |tick, ns|
+            tick[ns] = Hash.new do |ns, origin|
+              ns[origin] = []
+            end
+          end
+        end
+      end
+
+      # @group Methods for subclasses
+
+      # Feed data from the source.
+      #
+      # @param [String] ns  The namespace for this value.
+      # @param [String] origin The origin of this value.
+      # @param [Fixnum,Float] ts  The timestamp of this value.
+      # @param [Fixnum,Float] value  The actual value.
+      # @return [void]
+      def feed(ns, origin, ts, value)
+        raise NotImplementedError
+      end
+
+      # Flush and consolidate the buffer for the given +tick+.
+      #
+      # @param [Fixnum] tick  The tick to flush.
+      # @yield [ns, values]  Run the block once for every namespace, passing in
+      #                      all the consolidated values.
+      # @return [void]
+      def flush(tick, &block)
+        raise NotImplementedError
+      end
+
+      # @endgroup
+
+      # Add a value to all tick buffers. Usually called from {#feed}.
+      #
+      # @param [String] ns  The namespace for this value.
+      # @param [String] origin The origin of this value.
+      # @param [Object] value  Datasource specific value object.
+      # @return [void]
+      def add(ns, origin, value)
+        @ticks.each do |tick|
+          @buffer[tick][ns][origin] << value
+        end
+      end
+
+      # Atomically create a new buffer for +tick+ and iterate over the old one.
+      # Typically used by {#flush} to consolidate the samples.
+      #
+      # @param [Fixnum] tick  The tick buffer to flush.
+      # @yield [ns, origin, values]  Run the block once for every namespace
+      #                              and passing in all the consolidated
+      #                              values.
+      def each(tick, &block)
+        {}.tap do |result|
+          @mutex.synchronize do
+            result.replace(@buffer[tick].dup)
+            @buffer[tick].clear
+          end
+        end.each do |ns, origins|
+          origins.each do |origin, values|
+            yield ns, origin, values
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ganymed/sampler/derive.rb

@@ -0,0 +1,36 @@
+require 'ganymed/sampler/datasource'
+
+module Ganymed
+  class Sampler
+
+    ##
+    # A Derive {DataSource data source} will store the derivative of the line
+    # going from the last to the current value of the data source. Derive is
+    # similar to a {Counter} {DataSource data source} in that it produces a
+    # rate/s gauge, but only accepts absolute values and produces the
+    # derivative instead of summation of all samples.
+    #
+    # In order to get (meaningful) realtime events from a Derive {DataSource}
+    # samples must be sent multiple times per second. Derive does not store the
+    # previously flushed value for performance reasons and therefore cannot
+    # infer a rate/s gauge with only one event per flush.
+    #
+    class Derive < DataSource
+      def flush(tick, &block)
+        each(tick) do |ns, origin, events|
+          values = events.sort_by do |event| # sort by timestamp
+            event[0]
+          end.each_cons(2).map do |a, b| # derive each consective sample
+            (b[1] - a[1])/(b[0] - a[0]) # ∆value / ∆ts
+          end
+
+          yield ns, origin, values
+        end
+      end
+
+      def feed(ns, origin, ts, value)
+        add(ns, origin, [ts, value])
+      end
+    end
+  end
+end

data/lib/ganymed/sampler/gauge.rb

@@ -0,0 +1,24 @@
+require 'ganymed/sampler/datasource'
+
+module Ganymed
+  class Sampler
+
+    ##
+    # A Gauge is the simplest DataSource type. It simply records the given
+    # value in the buffer and emits all values in the buffer upon flush
+    # assuming the given values are in gauge-style (e.g. free memory, users
+    # currently logged in, etc).
+    #
+    class Gauge < DataSource
+      def flush(tick, &block)
+        each(tick) do |ns, origin, values|
+          yield ns, origin, values
+        end
+      end
+
+      def feed(ns, origin, ts, value)
+        add(ns, origin, value)
+      end
+    end
+  end
+end

data/lib/ganymed/sampler.rb

@@ -0,0 +1,106 @@
+require 'active_support/inflector'
+require 'eventmachine'
+require 'madvertise/ext/enumerable'
+require 'msgpack'
+require 'socket'
+
+require 'ganymed'
+
+module Ganymed
+
+  ##
+  # The Sampler processes samples from a UDP socket, stores these samples into
+  # memory and flushes a possibly interpolated and consolidated value to the
+  # {Processor} for every tick.
+  #
+  class Sampler
+    def initialize(config)
+      @config = config
+      @ticks = [1, config.resolution]
+      listen! and emit! and tick!
+    end
+
+    def listen!
+      @config.sampler.listen.tap do |listen|
+        log.info("processing samples on udp##{listen.host}:#{listen.port}")
+        EM.open_datagram_socket(listen.host, listen.port, Connection) do |connection|
+          connection.datasources = datasources
+        end
+      end
+    end
+
+    def emit!
+      @config.sampler.emit.tap do |emit|
+        log.info("emitting consolidated samples to udp##{emit.host}:#{emit.port}")
+        @client = Client.new(:processor => emit)
+        @processor = @client.processor
+      end
+    end
+
+    def tick!
+      @ticks.each do |tick|
+        EM.add_periodic_timer(tick) { flush(tick) }
+      end
+    end
+
+    def datasources
+      @datasources ||= Hash.new do |hsh, key|
+        klass = "Ganymed::Sampler::#{key.classify}".constantize
+        hsh[key] = klass.new(@ticks)
+      end
+    end
+
+    def flush(tick)
+      modifiers = tick == 1 ? ['realtime'] : []
+      datasources.each do |_, datasource|
+        datasource.flush(tick) do |ns, origin, values|
+          emit(ns, modifiers, origin, values)
+        end
+      end
+    end
+
+    # Consolidation functions.
+    CFS = {
+      :min => [:min],
+      :max => [:max],
+      :avg => [:mean],
+      :stdev => [:stdev],
+      :u90 => [:percentile, 0.9],
+    }
+
+    def emit(ns, modifiers, origin, values)
+      CFS.each do |cf, args|
+        begin
+          @processor.event(ns, values.send(*args), {
+            :cf => cf,
+            :origin => origin,
+            :modifiers => modifiers,
+          })
+        rescue Exception => exc
+          log.warn("failed to emit #{ns}@#{origin}")
+          log.exception(exc)
+        end
+      end
+    end
+
+    # @private
+    module Connection
+      attr_accessor :datasources
+
+      def receive_data(data)
+        begin
+          # pack format: <datasource><namespace><origin><timestamp><value>
+          data = data.unpack("Z*Z*Z*GG")
+          datasource = datasources[data.slice!(0)]
+          datasource.feed(*data)
+        rescue Exception => exc
+          log.exception(exc)
+        end
+      end
+    end
+  end
+end
+
+Dir[File.join(Ganymed::LIB_DIR, 'ganymed/sampler/*.rb')].each do |f|
+  require f
+end

data/lib/ganymed/version.rb

@@ -0,0 +1,3 @@
+module Ganymed
+  VERSION = '0.1.0'
+end