ganymed 0.2.3 → 0.3.0

data/Gemfile

@@ -3,6 +3,7 @@ source :rubygems
 gemspec
 
 group :development, :test do
+  gem 'ganymed-client', :path => '../ganymed-client'
   gem 'bson_ext'
   gem 'bundler'
   gem 'debugger'

data/ganymed.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   # master
   s.add_dependency "activesupport", ">= 3.2"
   s.add_dependency "eventmachine", ">= 0.12.10"
-  s.add_dependency "ganymed-client"
+  s.add_dependency "ganymed-client", ">= 0.2.0"
   s.add_dependency "madvertise-ext", ">= 0.1.2"
   s.add_dependency "madvertise-logging", ">= 0.3.2"
   s.add_dependency "mixlib-cli"

data/lib/ganymed/collector.rb

@@ -49,7 +49,17 @@ module Ganymed
     # @param [Section] config  The configuration object.
     def initialize(config)
       @config = config
-      @client = Client.new(@config.client)
+
+      @config.client.sampler.tap do |sampler|
+        log.info("emitting collected samples to tcp##{sampler.host}:#{sampler.port}")
+        @sampler = Ganymed::Client::Sampler.connect(sampler.host, sampler.port)
+      end
+
+      @config.client.processor.tap do |processor|
+        log.info("emitting collected events to tcp##{processor.host}:#{processor.port}")
+        @processor = Ganymed::Client::Processor.connect(processor.host, processor.port)
+      end
+
       load_collectors
     end
 
@@ -58,10 +68,9 @@ module Ganymed
       collectors.each do |file|
         name = File.basename(file, '.rb')
         config = @config.collectors[name.to_sym] || Section.new
-        config[:resolution] = @config.resolution
 
         log.debug("loading collector #{name} from #{file}")
-        Plugin.new(config, @client).from_file(file).tap do |collector|
+        Plugin.new(config, @sampler, @processor).from_file(file).tap do |collector|
           log.info("collecting #{name} metrics every #{collector.interval} seconds")
           collector.run
         end
@@ -97,21 +106,19 @@ module Ganymed
       attr_accessor :interval
 
       # Processor socket.
-      # @return [Ganymed::Client::ProcessorSocket]
+      # @return [Ganymed::Client::Processor]
       attr_accessor :processor
 
       # Sampler socket.
-      # @return [Ganymed::Client::SamplerSocket]
+      # @return [Ganymed::Client::Sampler]
       attr_accessor :sampler
 
       # Create a new plugin instance.
       #
       # @param [Section] config  The configuration object.
       # @param [Ganymed::Client] client  A client instance.
-      def initialize(config, client)
-        @config, @client = config, client
-        @processor = @client.processor
-        @sampler = @client.sampler
+      def initialize(config, sampler, processor)
+        @config, @sampler, @processor = config, sampler, processor
       end
 
       # Set the block used to collect metrics with this plugin.
@@ -119,9 +126,7 @@ module Ganymed
       # @param [Fixnum,Float] interval  Custom plugin interval.
       # @return [void]
       def collect(interval=nil, &block)
-        @interval ||= interval
-        @interval ||= @config.interval.tap{}
-        @interval ||= @config.resolution
+        @interval = interval || config.interval.tap{} || 1
         @collector = Proc.new(&block)
       end
 
@@ -129,7 +134,7 @@ module Ganymed
       # interval.
       # @return [void]
       def run
-        EM.add_periodic_timer(@interval) do
+        EM.add_periodic_timer(interval) do
           EM.defer { collect! }
         end
       end

data/lib/ganymed/collectors/cpu.rb

@@ -1,19 +1,21 @@
-collect(0.3) do
+collect do
   break if not File.readable?('/proc/stat')
 
-  File.open('/proc/stat').each do |line|
-    next if not line =~ /^cpu /
+  File.open('/proc/stat') do | file|
+    file.each do |line|
+      next if not line =~ /^cpu /
 
-    cpu = line.chomp.split[1,7].map do |x|
-      x.to_i / 100
-    end
+      cpu = line.chomp.split[1,7].map do |x|
+        x.to_i / 100
+      end
 
-    sampler.emit(:derive, "os.cpu.user", cpu[0])
-    sampler.emit(:derive, "os.cpu.nice", cpu[1])
-    sampler.emit(:derive, "os.cpu.system", cpu[2])
-    sampler.emit(:derive, "os.cpu.idle", cpu[3])
-    sampler.emit(:derive, "os.cpu.iowait", cpu[4])
-    sampler.emit(:derive, "os.cpu.irq", cpu[5])
-    sampler.emit(:derive, "os.cpu.softirq", cpu[6])
+      sampler.emit(:derive, "os.cpu.user", cpu[0])
+      sampler.emit(:derive, "os.cpu.nice", cpu[1])
+      sampler.emit(:derive, "os.cpu.system", cpu[2])
+      sampler.emit(:derive, "os.cpu.idle", cpu[3])
+      sampler.emit(:derive, "os.cpu.iowait", cpu[4])
+      sampler.emit(:derive, "os.cpu.irq", cpu[5])
+      sampler.emit(:derive, "os.cpu.softirq", cpu[6])
+    end
   end
 end

data/lib/ganymed/collectors/disk.rb

@@ -19,7 +19,7 @@ collect do
       processor.event("os.disk.#{name}.blocks", block_pc)
 
       files_pc = 1.0 - (st.files_free.to_f / st.files.to_f)
-      processor.event("os.disk.#{name}.files", block_pc)
+      processor.event("os.disk.#{name}.files", files_pc)
     end
   end
 end

data/lib/ganymed/collectors/iostat.rb

@@ -4,13 +4,15 @@ Struct.new("IOStat",
            :wio, :wmerge, :wsect, :wuse,
            :running, :use, :aveq)
 
-collect(0.3) do
+collect do
   next if not File.readable?('/proc/diskstats')
-  File.open('/proc/diskstats').each do |line|
-    ios = Struct::IOStat.new(*line.strip.split(/\s+/))
-    next if config.skip_numbered.tap{} and ios.dev =~ /\d+$/
-    next if config.exclude.map {|e| Regexp.new(e).match(ios.dev)}.any?
-    sampler.emit(:derive, "os.iostat.#{ios.dev}.rsect", ios.rsect)
-    sampler.emit(:derive, "os.iostat.#{ios.dev}.wsect", ios.wsect)
+  File.open('/proc/diskstats') do |file|
+    file.each do |line|
+      ios = Struct::IOStat.new(*line.strip.split(/\s+/))
+      next if config.skip_numbered.tap{} and ios.dev =~ /\d+$/
+      next if config.exclude.map {|e| Regexp.new(e).match(ios.dev)}.any?
+      sampler.emit(:derive, "os.iostat.#{ios.dev}.rsect", ios.rsect)
+      sampler.emit(:derive, "os.iostat.#{ios.dev}.wsect", ios.wsect)
+    end
   end
 end

data/lib/ganymed/collectors/load.rb

@@ -1,7 +1,7 @@
-collect(5) do
+collect do
   next if not File.readable?('/proc/loadavg')
-  File.open('/proc/loadavg') do |f|
-    loadavg = f.read.chomp.split[0,3].map(&:to_f)
+  File.open('/proc/loadavg') do |file|
+    loadavg = file.read.chomp.split[0,3].map(&:to_f)
     sampler.emit(:gauge, "os.loadavg", loadavg[0])
   end
 end

data/lib/ganymed/collectors/memory.rb

@@ -1,39 +1,41 @@
-collect(5) do
+collect do
   next if not File.readable?('/proc/meminfo')
 
   # calculate app memory from total
   apps = 0
 
-  File.open('/proc/meminfo').each do |line|
-    key, value, unit = line.chomp.split
-    key.gsub!(/:/, '')
-    value = value.to_i * 1024
+  File.open('/proc/meminfo') do |file|
+    file.each do |line|
+      key, value, unit = line.chomp.split
+      key.gsub!(/:/, '')
+      value = value.to_i * 1024
 
-    case key
-    when "MemTotal"
-      apps = value
-    when "MemFree"
-      sampler.emit(:gauge, "os.mem.unused", value)
-      apps -= value
-    when "Buffers"
-      sampler.emit(:gauge, "os.mem.buffers", value)
-      apps -= value
-    when "Cached"
-      sampler.emit(:gauge, "os.mem.cache", value)
-      apps -= value
-    when "SwapCached"
-      sampler.emit(:gauge, "os.mem.swap_cache", value)
-      apps -= value
-    when "Slab"
-      sampler.emit(:gauge, "os.mem.slab_cache", value)
-      apps -= value
-    when "PageTables"
-      sampler.emit(:gauge, "os.mem.page_tables", value)
-      apps -= value
-    when "Mapped"
-      sampler.emit(:gauge, "os.mem.mapped", value)
+      case key
+      when "MemTotal"
+        apps = value
+      when "MemFree"
+        sampler.emit(:gauge, "os.mem.unused", value)
+        apps -= value
+      when "Buffers"
+        sampler.emit(:gauge, "os.mem.buffers", value)
+        apps -= value
+      when "Cached"
+        sampler.emit(:gauge, "os.mem.cache", value)
+        apps -= value
+      when "SwapCached"
+        sampler.emit(:gauge, "os.mem.swap_cache", value)
+        apps -= value
+      when "Slab"
+        sampler.emit(:gauge, "os.mem.slab_cache", value)
+        apps -= value
+      when "PageTables"
+        sampler.emit(:gauge, "os.mem.page_tables", value)
+        apps -= value
+      when "Mapped"
+        sampler.emit(:gauge, "os.mem.mapped", value)
+      end
     end
-  end
 
-  sampler.emit(:gauge, "os.mem.apps", apps)
+    sampler.emit(:gauge, "os.mem.apps", apps)
+  end
 end

data/lib/ganymed/collectors/network.rb

@@ -3,11 +3,11 @@ Struct.new("NetworkStat", :dev,
            :rbytes, :rpackets, :rerrs, :rdrop, :rfifo, :rframe, :rcompressed, :rmulticast,
            :wbytes, :wpackets, :werrs, :wdrop, :wfifo, :wcolls, :wcarrier, :wcompressed)
 
-collect(0.3) do
+collect do
   break if not File.readable?('/proc/net/dev')
 
-  File.open('/proc/net/dev') do |f|
-    f.each do |line|
+  File.open('/proc/net/dev') do |file|
+    file.each do |line|
       next unless line =~ /:/
 
       ns = Struct::NetworkStat.new(*line.strip.split(/\s+/))

data/lib/ganymed/collectors/process.rb

@@ -1,16 +1,18 @@
-collect(0.3) do
+collect do
   break if not File.readable?('/proc/stat')
-  File.open('/proc/stat').each do |line|
-    key, value = line.chomp.split
-    case key
-    when "ctxt"
-      sampler.emit(:derive, "os.procs.switch", value.to_i)
-    when "processes"
-      sampler.emit(:derive, "os.procs.forks", value.to_i)
-    when "procs_running"
-      sampler.emit(:gauge, "os.procs.running", value.to_i)
-    when "procs_blocked"
-      sampler.emit(:gauge, "os.procs.blocked", value.to_i)
+  File.open('/proc/stat') do |file|
+    file.each do |line|
+      key, value = line.chomp.split
+      case key
+      when "ctxt"
+        sampler.emit(:derive, "os.procs.switch", value.to_i)
+      when "processes"
+        sampler.emit(:derive, "os.procs.forks", value.to_i)
+      when "procs_running"
+        sampler.emit(:gauge, "os.procs.running", value.to_i)
+      when "procs_blocked"
+        sampler.emit(:gauge, "os.procs.blocked", value.to_i)
+      end
     end
   end
 end

data/lib/ganymed/collectors/uptime.rb

@@ -1,8 +1,10 @@
 collect do
   break if not File.readable?('/proc/stat')
-  File.open('/proc/stat').each do |line|
-    next if not line =~ /^btime /
-    boottime = Time.at(line.chomp.split[1].to_i)
-    processor.event("os.reboot", 1, :now => boottime)
+  File.open('/proc/stat') do |file|
+    file.each do |line|
+      next if not line =~ /^btime /
+      boottime = Time.at(line.chomp.split[1].to_i)
+      processor.event("os.reboot", 1, :now => boottime)
+    end
   end
 end

data/lib/ganymed/config.yml

@@ -1,6 +1,4 @@
 generic:
-  resolution: 300
-
   eventmachine:
     threadpool_size: 2
 
@@ -24,10 +22,16 @@ generic:
 
   sampler:
     enabled: true
-    ticks: [1]
+    window: 300
     listen:
       host: 127.0.0.1
       port: 1337
+    consolidations:
+      min: [min]
+      max: [max]
+      avg: [mean]
+      stdev: [stdev]
+      u90: [percentile, 0.9]
 
   client:
     processor:

data/lib/ganymed/ext/array.rb

@@ -0,0 +1,38 @@
+require 'inline'
+
+class Array
+  # this is incredibly faster than doing ary.inject(0, :+)
+  # should only be called on numbers-only arrays, though ;-)
+  inline :C do |builder|
+    builder.c_raw <<-EOC
+    static VALUE sum(int argc, VALUE *argv, VALUE self) {
+      double result = 0;
+      VALUE *arr = RARRAY_PTR(self);
+      long i, len = RARRAY_LEN(self);
+
+      for (i = 0; i < len; i++) {
+        result += NUM2DBL(arr[i]);
+      }
+
+      return rb_float_new(result);
+    }
+    EOC
+  end
+
+  def mean
+    sum.to_f / length
+  end
+
+  def variance
+    m = mean
+    reduce(0) {|accum, item| accum + (item - m) ** 2}.to_f / (length - 1)
+  end
+
+  def stdev
+    Math.sqrt(variance)
+  end
+
+  def percentile(pc)
+    sort[(pc * length).ceil - 1]
+  end
+end

data/lib/ganymed/master.rb

@@ -31,74 +31,77 @@ module Ganymed
     end
 
     def run
-      unless EventMachine.reactor_running?
-        @reactor.join if @reactor.is_a?(Thread)
-        log.info("starting Ganymed reactor in #{Env.mode} mode")
-
-        # configure eventmachine
-        log.debug("EventMachine threadpool_size=#{config.eventmachine.threadpool_size}")
-        EventMachine.threadpool_size = config.eventmachine.threadpool_size
-
-        @reactor = Thread.new do
-          begin
-            EventMachine.run { setup_reactor }
-          rescue Exception => exc
-            log.exception(exc)
-          end
-        end
-      end
-
-      if config.profiling.gcprofiler
-        puts GC::Profiler.report
+      if pid = Process.fork
+        @supervisor = true
+        Process.wait(pid)
+      else
+        child
       end
     end
 
-    def setup_reactor
-      Processor.new(config) if config.processor.enabled
-      Sampler.new(config) if config.sampler.enabled
-      Collector.new(config) if config.collectors.any?
+    def child
+      log.info("starting Ganymed reactor in #{Env.mode} mode")
+
+      # configure eventmachine
+      log.debug("EventMachine threadpool_size=#{config.eventmachine.threadpool_size}")
+      EventMachine.threadpool_size = config.eventmachine.threadpool_size
+
+      EventMachine.epoll # use epoll
+      EventMachine.run do
+        Processor.new(config) if config.processor.enabled
+        Sampler.new(config) if config.sampler.enabled
+        Collector.new(config) if config.collectors.any?
+      end
+    rescue Exception => exc
+      log.exception(exc)
     end
 
     def before_starting
-      if config.profiling.perftools
-        log.info("activating PerfTools CPU profiler")
-        require 'perftools'
-        PerfTools::CpuProfiler.start('profile')
-      end
+      unless @supervisor
+        if config.profiling.perftools
+          log.info("activating PerfTools CPU profiler")
+          require 'perftools'
+          PerfTools::CpuProfiler.start('profile')
+        end
 
-      if config.profiling.gcprofiler
-        log.info("activating GC::Profiler")
-        GC::Profiler.enable
-      end
+        if config.profiling.gcprofiler
+          log.info("activating GC::Profiler")
+          GC::Profiler.enable
+        end
 
-      if config.profiling.rubyprof
-        require 'ruby-prof'
-        log.info("activating RubyProf")
-        RubyProf.start
+        if config.profiling.rubyprof
+          require 'ruby-prof'
+          log.info("activating RubyProf")
+          RubyProf.start
+        end
       end
     end
 
+    def before_stopping
+      if EventMachine.reactor_running?
+        log.info("shutting down Ganymed reactor")
+        EventMachine.stop_event_loop
+      end
+    end
 
     def after_stopping
-      log.info("shutting down Ganymed reactor")
-      EventMachine.stop_event_loop if EventMachine.reactor_running?
-      @reactor.join if @reactor.is_a?(Thread)
-
-      if config.profiling.perftools
-        PerfTools::CpuProfiler.stop
-        `pprof.rb --svg profile > profile.svg`
-      end
+      unless @supervisor
+        if config.profiling.perftools
+          PerfTools::CpuProfiler.stop
+          `pprof.rb --svg profile > profile.svg`
+        end
 
-      if config.profiling.rubyprof
-        result = RubyProf.stop
-        result.eliminate_methods!([
-          /Queue#pop/,
-          /Queue#push/,
-          /Mutex#synchronize/,
-          /Mutex#sleep/,
-        ])
-        printer = RubyProf::MultiPrinter.new(result)
-        printer.print(:path => "prof", :profile => "ganymed")
+        if config.profiling.rubyprof
+          result = RubyProf.stop
+          result.eliminate_methods!([
+            /Queue#pop/,
+            /Queue#push/,
+            /Mutex#synchronize/,
+            /Mutex#sleep/,
+          ])
+          printer = RubyProf::MultiPrinter.new(result)
+          printer.print(:path => "prof", :profile => "ganymed")
+        end
       end
 
       log.info("done shutting down. exiting ...")

data/lib/ganymed/processor.rb

@@ -9,7 +9,7 @@ require 'ganymed/websocket'
 module Ganymed
 
   ##
-  # The Processor accept {Event events} from a UDP socket and stores these
+  # The Processor accepts {Event events} from a UDP socket and stores these
   # metrics into MongoDB. All metrics are also published to interested parties
   # via WebSocket.
   #
@@ -18,7 +18,14 @@ module Ganymed
       @config = config
       @db = MongoDB.new(config.processor.mongodb)
       @websocket = Websocket.new(config.processor.websocket, @db)
-      @client = Client.new(:sampler => @config.client.sampler)
+
+      @config.client.sampler.tap do |sampler|
+        # schedule connect on next tick so that our sampler socket is
+        # available. Processor#initialize is called from EM.run and therefore
+        # no sockets have been opened yet.
+        EM.next_tick { @sampler = Ganymed::Client::Sampler.connect(sampler.host, sampler.port) }
+      end
+
       cache!
       listen!
     end
@@ -39,21 +46,28 @@ module Ganymed
         EM.open_datagram_socket(listen.host, listen.port, Connection) do |connection|
           connection.processor = self
         end
+
+        log.info("processing metrics on tcp##{listen.host}:#{listen.port}")
+        EM.start_server(listen.host, listen.port, Connection) do |connection|
+          connection.processor = self
+        end
       end
     end
 
     def process(data)
-      data = MessagePack.unpack(data)
       case data['_type'].to_sym
       when :event
         process_event(Event.parse(data))
-        @client.sampler.emit(:counter, 'ganymed.processor.events', 1)
       when :metadata
         process_metadata(data)
       end
+    rescue Exception => exc
+      log.exception(exc)
     end
 
     def process_event(event)
+      @sampler.emit(:counter, 'ganymed.processor.events', 1) rescue nil
+
       if not @metadata[event.origin].include?(event.ns)
         # insert origin + namespace into metadata
         metadata = @db['metadata'].find_and_modify({
@@ -70,19 +84,11 @@ module Ganymed
         @websocket.send(:metadata, [metadata])
       end
 
-      # send the event to websocket clients
-      @websocket.each do |connection|
-        connection.publish(event)
-      end
-
       # normalize timestamp for events with a resolution
       if event.resolution > 1
         event.timestamp -= event.timestamp % event.resolution
       end
 
-      # skip events that are produced below the resolution threshold
-      return if (1...@config.resolution).include?(event.resolution)
-
       # insert event into ns collection
       @db.collection(event.ns).update({
         :c => event.cf,
@@ -91,6 +97,11 @@ module Ganymed
       }, {
         :$set => {:v => event.value}
       }, :upsert => true)
+
+      # send the event to websocket clients
+      @websocket.each do |connection|
+        connection.publish(event)
+      end
     end
 
     def process_metadata(data)
@@ -105,17 +116,21 @@ module Ganymed
     end
 
     # @private
-    module Connection
+    class Connection < EM::Connection
       attr_accessor :processor
 
+      def initialize
+        @pac = MessagePack::Unpacker.new
+      end
+
       def receive_data(data)
-        EM.defer do
-          begin
-            processor.process(data)
-          rescue Exception => exc
-            log.exception(exc)
-          end
+        @pac.feed(data)
+        @pac.each do |obj|
+          EM.defer { processor.process(obj) }
         end
+      rescue Exception => exc
+        log.exception(exc)
+        close_connection
       end
     end
   end

data/lib/ganymed/sampler/counter.rb

@@ -1,3 +1,4 @@
+require 'ganymed/ext/array'
 require 'ganymed/sampler/datasource'
 
 module Ganymed
@@ -8,25 +9,11 @@ module Ganymed
     # as a webserver request, user login, etc) and produces a rate/s gauge.
     #
     class Counter < DataSource
-      def flush(tick, &block)
-        each(tick) do |ns, origin, events|
-          values = events.sort_by do |event| # sort by timestamp
-            event[0]
-          end.group_by do |event| # group by timestamp
-            event[0].to_i
-          end.map do |_, events| # calculate counts for each second.
-            events.reduce(0) do |memo, event| # sum(event.count)
-              memo + event[1]
-            end
-          end
-
-          yield ns, origin, values
+      def flush
+        each do |ns, origin, ts|
+          yield ns, origin, ts.values.map(&:sum)
         end
       end
-
-      def feed(ns, origin, ts, value)
-        add(ns, origin, [ts, value])
-      end
     end
   end
 end

data/lib/ganymed/sampler/datasource.rb

@@ -1,10 +1,13 @@
+require 'inline'
+
 module Ganymed
   class Sampler
 
     ##
     # A DataSource processes samples from various sources.  Samples are stored
-    # to an in-memory buffer and flushed to the {Processor} once for every
-    # tick.
+    # to an in-memory buffer using a sliding-window algorithm. Values are kept
+    # for a configurable time and consolidated values can be flushed to the
+    # processor at any time.
     #
     # All DataSources interpolate their samples to gauge style values before
     # flushing to the processor, e.g. a counter (like requests, or bytes sent
@@ -13,79 +16,69 @@ module Ganymed
     #
     class DataSource
 
-      attr_reader :ticks, :buffer
+      attr_reader :buffer, :window
 
-      # Create a new DataSource with buffers for every +tick+ in +ticks+.
-      #
-      # @param [Array] ticks  All ticks that should be stored in the buffer.
+      # Create a new DataSource.
       #
-      def initialize(ticks)
-        @ticks = ticks
-        @mutex = Mutex.new
-        @buffer = Hash.new do |buffer, tick|
-          buffer[tick] = Hash.new do |tick, ns|
-            tick[ns] = Hash.new do |ns, origin|
-              ns[origin] = []
+      # @param [Fixnum] window  Sliding window size.
+      def initialize(window)
+        @window = window
+        @buffer = Hash.new do |buffer, origin|
+          buffer[origin] = Hash.new do |origin, ns|
+            origin[ns] = Hash.new do |ns, timestamp|
+              ns[timestamp] = []
             end
           end
         end
       end
 
-      # @group Methods for subclasses
-
       # Feed data from the source.
       #
       # @param [String] ns  The namespace for this value.
       # @param [String] origin The origin of this value.
-      # @param [Time] ts  The timestamp of this value.
       # @param [Fixnum,Float] value  The actual value.
       # @return [void]
-      def feed(ns, origin, ts, value)
-        raise NotImplementedError
+      def feed(ns, origin, value)
+        @buffer[origin][ns][now] << value
       end
 
-      # Flush and consolidate the buffer for the given +tick+.
+      # Flush and consolidate the buffer.
       #
-      # @param [Fixnum] tick  The tick to flush.
-      # @yield [ns, values]  Run the block once for every namespace, passing in
-      #                      all the consolidated values.
+      # @yield [ns, origin, values]  Run the block once for every
+      #                              namespace/origin, passing in all the
+      #                              consolidated values.
       # @return [void]
-      def flush(tick, &block)
+      def flush
         raise NotImplementedError
       end
 
-      # @endgroup
-
-      # Add a value to all tick buffers. Usually called from {#feed}.
+      # Yield values for every origin/ns combination and remove old elements
+      # from the buffer (sliding window). This is used internally to iterate
+      # through the values in the current window.
       #
-      # @param [String] ns  The namespace for this value.
-      # @param [String] origin The origin of this value.
-      # @param [Object] value  Datasource specific value object.
-      # @return [void]
-      def add(ns, origin, value)
-        @ticks.each do |tick|
-          @buffer[tick][ns][origin] << value
+      # @yield [ns, origin, ts]  Run the block once for every namespace
+      #                          passing in all the collected samples grouped
+      #                          by timestamp.
+      def each(&block)
+        oldest = now - window
+        @buffer.each do |origin, ns|
+          ns.each do |ns, ts|
+            ts.delete_if { |ts, _| ts < oldest }
+            yield ns, origin, ts
+          end
         end
       end
 
-      # Atomically create a new buffer for +tick+ and iterate over the old one.
-      # Typically used by {#flush} to consolidate the samples.
-      #
-      # @param [Fixnum] tick  The tick buffer to flush.
-      # @yield [ns, origin, values]  Run the block once for every namespace
-      #                              and passing in all the consolidated
-      #                              values.
-      def each(tick, &block)
-        {}.tap do |result|
-          @mutex.synchronize do
-            result.replace(@buffer[tick].dup)
-            @buffer[tick].clear
-          end
-        end.each do |ns, origins|
-          origins.each do |origin, values|
-            yield ns, origin, values
-          end
-        end
+      # Time.now.utc.to_i is way too expensive
+      inline :C do |builder|
+        builder.include "<sys/time.h>"
+        builder.c <<-EOC
+        long now(void) {
+          struct timeval tv;
+          gettimeofday(&tv, NULL);
+          return tv.tv_sec;
+        }
+        EOC
       end
     end
   end

data/lib/ganymed/sampler/derive.rb

@@ -10,27 +10,18 @@ module Ganymed
     # rate/s gauge, but only accepts absolute values and produces the
     # derivative instead of summation of all samples.
     #
-    # In order to get (meaningful) realtime events from a Derive {DataSource}
-    # samples must be sent multiple times per second. Derive does not store the
-    # previously flushed value for performance reasons and therefore cannot
-    # infer a rate/s gauge with only one event per flush.
-    #
     class Derive < DataSource
-      def flush(tick, &block)
-        each(tick) do |ns, origin, events|
-          values = events.sort_by do |event| # sort by timestamp
-            event[0]
-          end.each_cons(2).map do |a, b| # derive each consective sample
-            (b[1] - a[1])/(b[0] - a[0]) # ∆value / ∆ts
+      def flush
+        each do |ns, origin, ts|
+          values = ts.each_cons(2).map do |a, b|
+            (b[1].last - a[1].last)/(b[0] - a[0]) # ∆value / ∆ts
+          end.reject do |value|
+            value < 0 # ignore overflow/counter wrap
           end
 
           yield ns, origin, values
         end
       end
-
-      def feed(ns, origin, ts, value)
-        add(ns, origin, [ts, value])
-      end
     end
   end
 end

data/lib/ganymed/sampler/gauge.rb

@@ -10,15 +10,11 @@ module Ganymed
     # currently logged in, etc).
     #
     class Gauge < DataSource
-      def flush(tick, &block)
-        each(tick) do |ns, origin, values|
-          yield ns, origin, values
+      def flush
+        each do |ns, origin, ts|
+          yield ns, origin, ts.values.flatten
         end
       end
-
-      def feed(ns, origin, ts, value)
-        add(ns, origin, value)
-      end
     end
   end
 end

data/lib/ganymed/sampler.rb

@@ -1,33 +1,29 @@
 require 'active_support/inflector'
 require 'eventmachine'
-require 'inline'
-require 'madvertise/ext/enumerable'
 
 require 'ganymed'
 require 'ganymed/client'
+require 'ganymed/ext/array'
 
 module Ganymed
 
   ##
   # The Sampler processes samples from a UDP socket, stores these samples into
   # memory and flushes a possibly interpolated and consolidated value to the
-  # {Processor} for every tick.
+  # {Processor}.
   #
   class Sampler
     def initialize(config)
       @config = config
-      @config.sampler.ticks |= [@config.resolution]
-      @fqdn = @config.fqdn
       listen!
       emit!
-      tick!
     end
 
     def listen!
       @config.sampler.listen.tap do |listen|
         log.info("processing samples on udp##{listen.host}:#{listen.port}")
         EM.open_datagram_socket(listen.host, listen.port, Connection) do |connection|
-          connection.fqdn = @fqdn
+          connection.fqdn = @config.fqdn
           connection.datasources = datasources
         end
       end
@@ -35,53 +31,37 @@ module Ganymed
 
     def emit!
       @config.client.processor.tap do |processor|
-        log.info("emitting consolidated samples to udp##{processor.host}:#{processor.port}")
-        @client = Client.new(:processor => processor)
-        @processor = @client.processor
+        log.info("emitting consolidated samples to tcp##{processor.host}:#{processor.port}")
+        @processor = Ganymed::Client::Processor.connect(processor.host, processor.port)
       end
-    end
 
-    def tick!
-      @config.sampler.ticks.each do |tick|
-        EM.add_periodic_timer(tick) { flush(tick) }
-      end
+      # emit consolidated samples 5 times per window
+      interval = [1, @config.sampler.window / 5].max
+      EM.add_periodic_timer(interval) { flush }
     end
 
     def datasources
       @datasources ||= Hash.new do |hsh, key|
         klass = "Ganymed::Sampler::#{key.classify}".constantize
-        hsh[key] = klass.new(@config.sampler.ticks)
+        hsh[key] = klass.new(@config.sampler.window)
       end
     end
 
-    def flush(tick)
+    def flush
       datasources.each do |_, datasource|
-        EM.defer do
-          datasource.flush(tick) do |ns, origin, values|
-            emit(ns, origin, tick, values)
-          end
+        datasource.flush do |ns, origin, values|
+          emit(ns, origin, values)
         end
       end
     end
 
-    # Consolidation functions.
-    CFS = {
-      :min => [:min],
-      :max => [:max],
-      :avg => [:mean],
-      :stdev => [:stdev],
-      :u90 => [:percentile, 0.9],
-    }
-
-    def emit(ns, origin, tick, values)
-      CFS.each do |cf, args|
-        # do not calculate the consolidations below threshold
-        next if tick < @config.resolution and cf != :avg
+    def emit(ns, origin, values)
+      @config.sampler.consolidations.each do |cf, args|
         begin
           @processor.event(ns, values.send(*args), {
             :cf => cf,
             :origin => origin,
-            :resolution => tick,
+            :resolution => @config.sampler.window,
           })
         rescue Exception => exc
           log.warn("failed to emit #{ns}@#{origin}")
@@ -94,24 +74,12 @@ module Ganymed
     module Connection
       attr_accessor :datasources, :fqdn
 
-      # Time.now.utc.to_f is way too expensive
-      inline :C do |builder|
-        builder.include "<sys/time.h>"
-        builder.c <<-EOC
-        double timestamp(void) {
-          struct timeval tv;
-          gettimeofday(&tv, NULL);
-          return ((double)tv.tv_sec + ((double)tv.tv_usec / 10000000.0));
-        }
-        EOC
-      end
-
       def receive_data(data)
         begin
-          # pack format: <datasource><namespace><origin><timestamp><value>
-          data = data.unpack("Z*Z*Z*GG")
+          # pack format: <datasource><namespace><origin><value>
+          data = data.unpack("Z*Z*Z*G")
           datasources[data.slice!(0)].feed(*data)
-          datasources['counter'].feed('ganymed.sampler.samples', @fqdn, timestamp, 1)
+          datasources['counter'].feed('ganymed.sampler.samples', fqdn, 1)
         rescue Exception => exc
           log.exception(exc)
         end

data/lib/ganymed/version.rb

@@ -1,4 +1,4 @@
 module Ganymed
   # @private
-  VERSION = '0.2.3'
+  VERSION = '0.3.0'
 end

data/lib/ganymed/websocket/query.rb

@@ -12,12 +12,16 @@ module Ganymed
         return if not authenticated?
         data.each do |ns, query|
           query_id = query.delete('_id')
+
           log.debug("query #{query_id} from #{peer}: #{ns}(#{query.inspect})")
+          t = Time.now
 
           events = db.collection(ns).find(query).map do |event|
             Event.parse(event.merge({'n' => ns}))
           end
 
+          log.debug("query #{query_id} returned #{events.length} results in #{Time.now - t}s")
+
           send(:result, {query_id => convert(events)})
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ganymed
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-30 00:00:00.000000000 Z
+date: 2012-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &5360620 !ruby/object:Gem::Requirement
+  requirement: &16400780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '3.2'
   type: :runtime
   prerelease: false
-  version_requirements: *5360620
+  version_requirements: *16400780
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &5358160 !ruby/object:Gem::Requirement
+  requirement: &16399280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,21 +32,21 @@ dependencies:
         version: 0.12.10
   type: :runtime
   prerelease: false
-  version_requirements: *5358160
+  version_requirements: *16399280
 - !ruby/object:Gem::Dependency
   name: ganymed-client
-  requirement: &5355500 !ruby/object:Gem::Requirement
+  requirement: &16397660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *5355500
+  version_requirements: *16397660
 - !ruby/object:Gem::Dependency
   name: madvertise-ext
-  requirement: &5353920 !ruby/object:Gem::Requirement
+  requirement: &16396540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *5353920
+  version_requirements: *16396540
 - !ruby/object:Gem::Dependency
   name: madvertise-logging
-  requirement: &5381900 !ruby/object:Gem::Requirement
+  requirement: &16395360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: 0.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *5381900
+  version_requirements: *16395360
 - !ruby/object:Gem::Dependency
   name: mixlib-cli
-  requirement: &5380600 !ruby/object:Gem::Requirement
+  requirement: &16394880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *5380600
+  version_requirements: *16394880
 - !ruby/object:Gem::Dependency
   name: RubyInline
-  requirement: &5378540 !ruby/object:Gem::Requirement
+  requirement: &16410420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *5378540
+  version_requirements: *16410420
 - !ruby/object:Gem::Dependency
   name: servolux
-  requirement: &5376720 !ruby/object:Gem::Requirement
+  requirement: &16408420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *5376720
+  version_requirements: *16408420
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &5375220 !ruby/object:Gem::Requirement
+  requirement: &16406820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *5375220
+  version_requirements: *16406820
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &5396280 !ruby/object:Gem::Requirement
+  requirement: &16405220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: '1.6'
   type: :runtime
   prerelease: false
-  version_requirements: *5396280
+  version_requirements: *16405220
 - !ruby/object:Gem::Dependency
   name: em-websocket
-  requirement: &5392360 !ruby/object:Gem::Requirement
+  requirement: &16403660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,10 +131,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *5392360
+  version_requirements: *16403660
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &5391160 !ruby/object:Gem::Requirement
+  requirement: &16402580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -142,10 +142,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *5391160
+  version_requirements: *16402580
 - !ruby/object:Gem::Dependency
   name: sys-filesystem
-  requirement: &5414000 !ruby/object:Gem::Requirement
+  requirement: &16433420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -153,10 +153,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *5414000
+  version_requirements: *16433420
 - !ruby/object:Gem::Dependency
   name: ohai
-  requirement: &5406240 !ruby/object:Gem::Requirement
+  requirement: &16432680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -164,7 +164,7 @@ dependencies:
         version: 0.6.12
   type: :runtime
   prerelease: false
-  version_requirements: *5406240
+  version_requirements: *16432680
 description: Ganymed is an event collection daemon
 email:
 - bb@xnull.de
@@ -197,6 +197,7 @@ files:
 - lib/ganymed/collectors/uptime.rb
 - lib/ganymed/config.yml
 - lib/ganymed/event.rb
+- lib/ganymed/ext/array.rb
 - lib/ganymed/master.rb
 - lib/ganymed/mongodb.rb
 - lib/ganymed/processor.rb
@@ -238,7 +239,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1127904829280483897
+      hash: 4174202345634086766
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -247,7 +248,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1127904829280483897
+      hash: 4174202345634086766
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.17