ganymed 0.1.1 → 0.1.2

data/.gitignore

@@ -14,6 +14,7 @@ lib/bundler/man
 node_modules/
 old/
 pkg
+prof/
 rdoc
 spec/reports
 test.js

data/ganymed.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_dependency "madvertise-logging", ">= 0.3.2"
   s.add_dependency "mixlib-cli"
   s.add_dependency "servolux"
+  s.add_dependency "RubyInline"
 
   # processor
   s.add_dependency "msgpack"

data/lib/ganymed/client.rb

@@ -65,7 +65,7 @@ module Ganymed
       # @param [Fixnum, Float] value  Sample value.
       # @param [Time] now  Sample timestamp.
       def emit(ds, ns, value, now=Time.now.utc)
-        data = [ds.to_s, ns, @origin, now.to_i + (now.usec * 1e-6), value.to_f]
+        data = [ds.to_s, ns, @origin, now.to_f, value.to_f]
         send(data.pack("Z*Z*Z*GG"))
       end
     end
@@ -82,23 +82,23 @@ module Ganymed
       # @param [Hash] opts  Options
       # @option opts [String] cf  Consolidation function used in this event.
       # @option opts [Time] now  {Event} timestamp.
+      # @option opts [Fixnum] resolution  {Event} resolution.
       # @option opts [String] origin  {Event} origin.
-      # @option opts [Array] modifiers  Event modifiers.
       def event(ns, value, opts={})
         opts = {
           :cf => nil,
           :now => Time.now.utc,
+          :resolution => 0,
           :origin => @origin,
-          :modifiers => [],
         }.merge(opts)
 
         {
           :_type => :event,
           :n => ns,
-          :m => opts[:modifiers],
           :c => opts[:cf],
           :o => opts[:origin],
           :t => opts[:now].to_i,
+          :r => opts[:resolution].to_i,
           :v => value
         }.tap do |data|
           send(data.to_msgpack)

data/lib/ganymed/collector.rb

@@ -21,8 +21,8 @@ module Ganymed
     # @param [Configuration] config  The configuration object.
     def initialize(config)
       @config = config
-      @client = Client.new(:processor => @config.collector.processor,
-                           :sampler => @config.collector.sampler)
+      @client = Client.new(:processor => @config.client.processor,
+                           :sampler => @config.client.sampler)
 
       load_collectors
       @config.collectors.each do |collector|

data/lib/ganymed/config.yml

@@ -24,15 +24,12 @@ generic:
 
   sampler:
     enabled: true
+    ticks: [1]
     listen:
       host: 127.0.0.1
       port: 1337
-    emit:
-      host: 127.0.0.1
-      port: 1336
 
-  collector:
-    enabled: true
+  client:
     processor:
       host: 127.0.0.1
       port: 1336

data/lib/ganymed/event.rb

@@ -4,11 +4,11 @@ module Ganymed
   # An incoming event is a hash-like object and contains the following keys:
   #
   # n::  namespace (e.g. +os.cpu.idle+, +nginx.requests+, etc)
-  # m::  event modifiers (e.g. +realtime+)
   # c::  a consolidation function used to build this event from multiple
   #      samples
   # o::  origin (e.g. +web1.example.com+, +myapplication+)
   # t::  timestamp as seconds since epoch
+  # r::  resolution of this event (i.e. how often is this event sent to the processor)
   # v::  the value in integer or floating-point representation
   #
   class Event
@@ -16,10 +16,6 @@ module Ganymed
     # @return [String]
     attr_accessor :ns
 
-    # The event modifiers.
-    # @return [Array]
-    attr_accessor :modifiers
-
     # The consolidation function used to build this event from multiple
     # samples.
     # @return [String]
@@ -33,6 +29,10 @@ module Ganymed
     # @return [Fixnum]
     attr_accessor :timestamp
 
+    # The event resolution
+    # @return [Fixnum]
+    attr_accessor :resolution
+
     # The actual value of this sample.
     # @return [Object]
     attr_accessor :value
@@ -43,10 +43,10 @@ module Ganymed
     def self.parse(obj)
       new.tap do |event|
         event.ns = obj['n']
-        event.modifiers = obj['m']
         event.cf = obj['c']
         event.origin = obj['o']
         event.timestamp = obj['t'].to_i
+        event.resolution = obj['r'].to_i
 
         value = obj['v']
         if value.is_a?(Float) and (value.nan? or value.infinite?)
@@ -73,20 +73,13 @@ module Ganymed
       }
     end
 
-    # Check if this event is a realtime event.
-    #
-    # @return [TrueClass,FalseClass]
-    def realtime?
-      @modifiers.include?("realtime")
-    end
-
     # Validate this event.
     def validate
       unless @ns =~ /^[a-z][a-z0-9_\.-]+[a-z0-9]$/
         raise "invalid namespace: #{@ns.inspect}"
       end
 
-      if @ns =~ /^(system|ganymed)\./
+      if @ns =~ /^(system\.|metadata)/
         raise "namespace is reserved: #{@ns.inspect}"
       end
 

data/lib/ganymed/master.rb

@@ -20,6 +20,9 @@ module Ganymed
       @default_config_file = File.join(LIB_DIR, 'ganymed/config.yml')
       @config_file = cli[:config_file]
 
+      # store fqdn for later use
+      config.fqdn = ::Socket.gethostbyname(::Socket.gethostname).first
+
       # initialize servolux
       super('ganymed',
             :interval => 1,
@@ -53,7 +56,7 @@ module Ganymed
     def setup_reactor
       Processor.new(config) if config.processor.enabled
       Sampler.new(config) if config.sampler.enabled
-      Collector.new(config) if config.collector.enabled
+      Collector.new(config) if config.collectors.any?
     end
 
     def before_starting

data/lib/ganymed/processor.rb

@@ -11,6 +11,7 @@ module Ganymed
       @config = config
       @db = MongoDB.new(config.processor.mongodb)
       @websocket = Websocket.new(config.processor.websocket, @db)
+      @client = Client.new(:sampler => @config.client.sampler)
       listen!
     end
 
@@ -18,75 +19,81 @@ module Ganymed
       @config.processor.listen.tap do |listen|
         log.info("processing metrics on udp##{listen.host}:#{listen.port}")
         EM.open_datagram_socket(listen.host, listen.port, Connection) do |connection|
-          connection.db = @db
-          connection.websocket = @websocket
+          connection.processor = self
         end
       end
     end
 
-    module Connection
-      attr_accessor :db, :websocket
-
-      def receive_data(data)
-        EM.defer do
-          begin
-            data = MessagePack.unpack(data)
-            case data['_type'].to_sym
-            when :event
-              process_event(data)
-            when :metadata
-              process_metadata(data)
-            end
-          rescue Exception => exc
-            log.exception(exc)
-          end
-        end
+    def process(data)
+      data = MessagePack.unpack(data)
+      case data['_type'].to_sym
+      when :event
+        process_event(Event.parse(data))
+        @client.sampler.emit(:counter, 'ganymed.processor.events', 1)
+      when :metadata
+        process_metadata(data)
       end
+    end
 
-      def process_event(data)
-        event = Event.parse(data)
+    def process_event(event)
+      # insert origin + namespace into metadata
+      metadata = @db['metadata'].find_and_modify({
+        query: {:_id => event.origin},
+        update: {:$addToSet => {:namespaces => event.ns}},
+        upsert: true,
+      })
 
-        # insert origin + namespace into metadata
-        metadata = db['metadata'].find_and_modify({
-          query: {:_id => event.origin},
-          update: {:$addToSet => {:namespaces => event.ns}},
-          upsert: true,
-        })
+      # set defaults in case find_and_modify returned an empty document
+      metadata['_id'] ||= event.origin
+      metadata['namespaces'] ||= []
 
-        # set defaults in case find_and_modify returned an empty document
-        metadata['_id'] ||= event.origin
-        metadata['namespaces'] ||= []
+      # if metadata has changed notify websocket clients
+      if not metadata['namespaces'].include?(event.ns)
+        metadata['namespaces'] |= [event.ns]
+        @websocket.send(:metadata, [metadata])
+      end
 
-        # if metadata has changed notify websocket clients
-        if not metadata['namespaces'].include?(event.ns)
-          metadata['namespaces'] |= [event.ns]
-          websocket.send(:metadata, [metadata])
-        end
+      # send the event to websocket clients
+      @websocket.each do |connection|
+        connection.publish(event)
+      end
 
-        # send the event to websocket clients
-        websocket.each do |connection|
-          connection.publish(event)
-        end
+      # skip events that are produced below the mongodb storage threshold
+      return if (1..@config.resolution).include?(event.resolution)
 
-        # do not store realtime events into mongodb
-        return if event.realtime?
+      # insert event into ns collection
+      @db.collection(event.ns).update({
+        :c => event.cf,
+        :o => event.origin,
+        :t => event.timestamp
+      }, {
+        :$set => {:v => event.value}
+      }, :upsert => true)
+    end
 
-        # insert event into ns collection
-        db.collection(event.ns).update({
-          :c => event.cf,
-          :o => event.origin,
-          :t => event.timestamp
-        }, {
-          :$set => {:v => event.value}
-        }, :upsert => true)
-      end
+    def process_metadata(data)
+      metadata = @db['metadata'].find_and_modify({
+        query: {:_id => data['origin']},
+        update: {:$set => {:data => data['data']}},
+        upsert: true,
+        new: true
+      })
+
+      @websocket.send(:metadata, [metadata])
+    end
 
-      def process_metadata(data)
-        db['metadata'].update({
-          :_id => data['origin'],
-        }, {
-          :$set => {:data => data['data']}
-        }, :upsert => true)
+    # @private
+    module Connection
+      attr_accessor :processor
+
+      def receive_data(data)
+        EM.defer do
+          begin
+            processor.process(data)
+          rescue Exception => exc
+            log.exception(exc)
+          end
+        end
       end
     end
   end

data/lib/ganymed/sampler.rb

@@ -3,6 +3,7 @@ require 'eventmachine'
 require 'madvertise/ext/enumerable'
 require 'msgpack'
 require 'socket'
+require 'inline'
 
 require 'ganymed'
 
@@ -16,7 +17,8 @@ module Ganymed
   class Sampler
     def initialize(config)
       @config = config
-      @ticks = [1, config.resolution]
+      @config.sampler.ticks |= [@config.resolution]
+      @fqdn = @config.fqdn
       listen! and emit! and tick!
     end
 
@@ -24,21 +26,22 @@ module Ganymed
       @config.sampler.listen.tap do |listen|
         log.info("processing samples on udp##{listen.host}:#{listen.port}")
         EM.open_datagram_socket(listen.host, listen.port, Connection) do |connection|
+          connection.fqdn = @fqdn
           connection.datasources = datasources
         end
       end
     end
 
     def emit!
-      @config.sampler.emit.tap do |emit|
-        log.info("emitting consolidated samples to udp##{emit.host}:#{emit.port}")
-        @client = Client.new(:processor => emit)
+      @config.client.processor.tap do |processor|
+        log.info("emitting consolidated samples to udp##{processor.host}:#{processor.port}")
+        @client = Client.new(:processor => processor)
         @processor = @client.processor
       end
     end
 
     def tick!
-      @ticks.each do |tick|
+      @config.sampler.ticks.each do |tick|
         EM.add_periodic_timer(tick) { flush(tick) }
       end
     end
@@ -46,16 +49,15 @@ module Ganymed
     def datasources
       @datasources ||= Hash.new do |hsh, key|
         klass = "Ganymed::Sampler::#{key.classify}".constantize
-        hsh[key] = klass.new(@ticks)
+        hsh[key] = klass.new(@config.sampler.ticks)
       end
     end
 
     def flush(tick)
-      modifiers = tick == 1 ? ['realtime'] : []
       datasources.each do |_, datasource|
         EM.defer do
           datasource.flush(tick) do |ns, origin, values|
-            emit(ns, modifiers, origin, values)
+            emit(ns, origin, tick, values)
           end
         end
       end
@@ -70,13 +72,15 @@ module Ganymed
       :u90 => [:percentile, 0.9],
     }
 
-    def emit(ns, modifiers, origin, values)
+    def emit(ns, origin, tick, values)
       CFS.each do |cf, args|
+        # do not calculate the consolidations below threshold
+        next if tick < @config.resolution and cf != :avg
         begin
           @processor.event(ns, values.send(*args), {
             :cf => cf,
             :origin => origin,
-            :modifiers => modifiers,
+            :resolution => tick,
           })
         rescue Exception => exc
           log.warn("failed to emit #{ns}@#{origin}")
@@ -87,14 +91,26 @@ module Ganymed
 
     # @private
     module Connection
-      attr_accessor :datasources
+      attr_accessor :datasources, :fqdn
+
+      # Time.now.utc.to_f is way too expensive
+      inline :C do |builder|
+        builder.include "<sys/time.h>"
+        builder.c <<-EOC
+        double timestamp(void) {
+          struct timeval tv;
+          gettimeofday(&tv, NULL);
+          return ((double)tv.tv_sec + ((double)tv.tv_usec / 10000000.0));
+        }
+        EOC
+      end
 
       def receive_data(data)
         begin
           # pack format: <datasource><namespace><origin><timestamp><value>
           data = data.unpack("Z*Z*Z*GG")
-          datasource = datasources[data.slice!(0)]
-          datasource.feed(*data)
+          datasources[data.slice!(0)].feed(*data)
+          datasources['counter'].feed('ganymed.sampler.samples', @fqdn, timestamp, 1)
         rescue Exception => exc
           log.exception(exc)
         end

data/lib/ganymed/version.rb

@@ -1,3 +1,3 @@
 module Ganymed
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ganymed
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-23 00:00:00.000000000 Z
+date: 2012-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &11641160 !ruby/object:Gem::Requirement
+  requirement: &12783700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '3.2'
   type: :runtime
   prerelease: false
-  version_requirements: *11641160
+  version_requirements: *12783700
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &11637760 !ruby/object:Gem::Requirement
+  requirement: &12781700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 0.12.10
   type: :runtime
   prerelease: false
-  version_requirements: *11637760
+  version_requirements: *12781700
 - !ruby/object:Gem::Dependency
   name: madvertise-ext
-  requirement: &11636500 !ruby/object:Gem::Requirement
+  requirement: &12780400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *11636500
+  version_requirements: *12780400
 - !ruby/object:Gem::Dependency
   name: madvertise-logging
-  requirement: &11634080 !ruby/object:Gem::Requirement
+  requirement: &12806280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *11634080
+  version_requirements: *12806280
 - !ruby/object:Gem::Dependency
   name: mixlib-cli
-  requirement: &11650400 !ruby/object:Gem::Requirement
+  requirement: &12805060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11650400
+  version_requirements: *12805060
 - !ruby/object:Gem::Dependency
   name: servolux
-  requirement: &11648860 !ruby/object:Gem::Requirement
+  requirement: &12803060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11648860
+  version_requirements: *12803060
+- !ruby/object:Gem::Dependency
+  name: RubyInline
+  requirement: &12800700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *12800700
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &11647780 !ruby/object:Gem::Requirement
+  requirement: &12815620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11647780
+  version_requirements: *12815620
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &11647060 !ruby/object:Gem::Requirement
+  requirement: &12812240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +109,10 @@ dependencies:
         version: '1.6'
   type: :runtime
   prerelease: false
-  version_requirements: *11647060
+  version_requirements: *12812240
 - !ruby/object:Gem::Dependency
   name: em-websocket
-  requirement: &11645980 !ruby/object:Gem::Requirement
+  requirement: &12810000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +120,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11645980
+  version_requirements: *12810000
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &11644260 !ruby/object:Gem::Requirement
+  requirement: &12807700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +131,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11644260
+  version_requirements: *12807700
 - !ruby/object:Gem::Dependency
   name: sys-filesystem
-  requirement: &11660200 !ruby/object:Gem::Requirement
+  requirement: &12823980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,10 +142,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11660200
+  version_requirements: *12823980
 - !ruby/object:Gem::Dependency
   name: ohai
-  requirement: &11658260 !ruby/object:Gem::Requirement
+  requirement: &12822240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -142,7 +153,7 @@ dependencies:
         version: 0.6.12
   type: :runtime
   prerelease: false
-  version_requirements: *11658260
+  version_requirements: *12822240
 description: Ganymed is an event collection daemon
 email:
 - bb@xnull.de
@@ -217,7 +228,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3485584446673225312
+      hash: -813213877420202571
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -226,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3485584446673225312
+      hash: -813213877420202571
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.17