ganymed 0.1.2 → 0.2.0

data/.gitignore

@@ -6,6 +6,7 @@
 Gemfile.lock
 InstalledFiles
 _yardoc
+config.yml
 coverage/
 doc/
 ganymed.pid
@@ -15,6 +16,7 @@ node_modules/
 old/
 pkg
 prof/
+profile*
 rdoc
 spec/reports
 test.js

data/README.md

@@ -2,18 +2,19 @@
 
 Ganymed is an event collection daemon.
 
-Ganymed processes system and application metrics in realtime and can easily
-handle thousands of samples per second.
+Ganymed processes system and application metrics and events in realtime and can
+easily handle thousands of samples per second. Ganymed provides these services
+to administrators and third-party applications:
 
-* the *sampler* accepts high frequency samples. these samples are then
-  consolidated based on the data source type and emitted to the *processor*.
+* The *Sampler* accepts high frequency samples. these samples are then
+  consolidated based on the data source type and emitted to the *Processor*.
 
-* the *processor* accepts low frequency data points (possibly created by the
-  *sampler*) and publishes these to interested parties via WebSocket in
-  realtime. All samples are also stored in MongoDB for later analysis.
+* The *Processor* accepts metrics (events) and stores these metrics into
+  MongoDB. All metrics are also published to interested parties via WebSocket.
 
-* the *collector* will send various system metrics (cpu, memory, network, etc)
-  to the *sampler* and *processor*
+* The *Collector* is a simple thread that polls various system-level metrics
+  that are not pushed to the *Sampler*/*Processor* from third-party
+  applications.
 
 ## Installation
 

data/ganymed.gemspec

@@ -19,11 +19,12 @@ Gem::Specification.new do |s|
   # master
   s.add_dependency "activesupport", ">= 3.2"
   s.add_dependency "eventmachine", ">= 0.12.10"
+  s.add_dependency "ganymed-client"
   s.add_dependency "madvertise-ext", ">= 0.1.2"
   s.add_dependency "madvertise-logging", ">= 0.3.2"
   s.add_dependency "mixlib-cli"
-  s.add_dependency "servolux"
   s.add_dependency "RubyInline"
+  s.add_dependency "servolux"
 
   # processor
   s.add_dependency "msgpack"

data/lib/ganymed.rb

@@ -1,4 +1,12 @@
+##
+# Ganymed is an event collection daemon.
+#
+# For more information about Ganymed go to the {file:README.md README}
+#
 module Ganymed
+  # @private
   GEM_ROOT = File.expand_path("../..", __FILE__).freeze
+
+  # @private
   LIB_DIR = File.join(GEM_ROOT, "lib").freeze
 end

data/lib/ganymed/collector.rb

@@ -6,96 +6,155 @@ require 'ganymed/client'
 module Ganymed
 
   ##
-  # The Collector polls various system-level metrics that are not pushed to the
-  # {Processor} automatically.
+  # The Collector is a simple thread that polls various system-level metrics
+  # that are not pushed to the {Sampler}/{Processor} from third-party
+  # applications.
+  #
+  # The Collector features a very simple DSL to implement custom collector
+  # plugins. Simply create a file with a +.rb+ file extension and a +collect+
+  # block:
+  #
+  #   collect do
+  #     File.open('/proc/foobar').each do |line|
+  #       parse(line)
+  #     end
+  #   end
+  #
+  # Copy this plugin to a location of your choice and set
+  # +collector.load_paths+ in your configuration to point to the containing
+  # directory.
+  #
+  # The collector thread will then call this plugin once per resolution tick.
+  # See {file:doc/configuration.md} for details.
+  #
+  # Custom intervals can also be specified for high frequency sampling:
+  #
+  #   collect(0.3) do
+  #     File.open('/proc/foobar').each do |line|
+  #       parse(line)
+  #     end
+  #   end
   #
   class Collector
     # The configuration object.
+    # @return [Section]
     attr_reader :config
 
-    # The {Client} object.
+    # The client object.
+    # @return [Ganymed::Client]
     attr_reader :client
 
     # Create a new collector instance and initialize all configured collectors.
     #
-    # @param [Configuration] config  The configuration object.
+    # @param [Section] config  The configuration object.
     def initialize(config)
       @config = config
-      @client = Client.new(:processor => @config.client.processor,
-                           :sampler => @config.client.sampler)
-
+      @client = Client.new(@config.client)
       load_collectors
-      @config.collectors.each do |collector|
-        log.info("initializing collector #{collector.klass.demodulize}")
-        collector.klass.constantize.new(collector, self).run
-      end
     end
 
+    # @private
     def load_collectors
-      # collectors shipped with ganymed
-      Dir[File.join(Ganymed::LIB_DIR, 'ganymed/collector/*.rb')].each do |f|
-        log.debug("Loading collector from #{f}")
-        require f
-      end
+      collectors.each do |file|
+        name = File.basename(file, '.rb')
+        config = @config.collectors[name.to_sym] || Section.new
+        config[:resolution] = @config.resolution
 
-      # custom collectors
-      if @config.collector.path.tap{}
-        Dir[File.join(@config.collector.path, '*.rb')].each do |f|
-          log.debug("Loading collector from #{f}")
-          require f
+        log.debug("loading collector #{name} from #{file}")
+        Plugin.new(config, @client).from_file(file).tap do |collector|
+          log.info("collecting #{name} metrics every #{collector.interval} seconds")
+          collector.run
         end
       end
     end
 
+    # @private
+    def collectors
+      load_paths = [@config.collector.load_paths.tap{}].flatten.compact
+      load_paths << File.join(Ganymed::LIB_DIR, 'ganymed/collectors')
+
+      [].tap do |files|
+        load_paths.each do |load_path|
+          log.debug("loading collectors from #{load_path}")
+          Dir[File.join(load_path, '*.rb')].each do |file|
+            files << file
+          end
+        end
+      end.sort.uniq
+    end
+
     ##
-    # A base class for collectors.
+    # Base class and DSL for {Collector} plugins.
     #
-    class Base
+    class Plugin
+
+      # The configuration object.
+      # @return [Section]
+      attr_accessor :config
+
+      # Plugin interval
+      # @return [Fixnum,Float]
+      attr_accessor :interval
 
-      # Create a new data collector instance.
+      # Processor socket.
+      # @return [Ganymed::Client::ProcessorSocket]
+      attr_accessor :processor
+
+      # Sampler socket.
+      # @return [Ganymed::Client::SamplerSocket]
+      attr_accessor :sampler
+
+      # Create a new plugin instance.
+      #
+      # @param [Section] config  The configuration object.
+      # @param [Ganymed::Client] client  A client instance.
+      def initialize(config, client)
+        @config, @client = config, client
+        @processor = @client.processor
+        @sampler = @client.sampler
+      end
+
+      # Set the block used to collect metrics with this plugin.
       #
-      # @param [Configuration] config  The configuration object.
-      # @param [Collector] collector  The collector instance.
-      def initialize(config, collector)
-        @config, @collector = config, collector
-        @processor = @collector.client.processor
-        @sampler = @collector.client.sampler
+      # @param [Fixnum,Float] interval  Custom plugin interval.
+      # @return [void]
+      def collect(interval=nil, &block)
+        @interval ||= interval
+        @interval ||= @config.interval.tap{}
+        @interval ||= @config.resolution
+        @collector = Proc.new(&block)
       end
 
       # Start the EventMachine timer to collect metrics at the specified
       # interval.
+      # @return [void]
       def run
-        timer do
-          begin
-            collect!
-          rescue Exception => e
-            log.exception(e)
-          end
+        EM.add_periodic_timer(@interval) do
+          EM.defer { collect! }
         end
       end
 
       # @private
-      def timer
-        if interval
-          EM.add_periodic_timer(interval) do
-            EM.defer { yield }
-          end
-        else
-          yield
+      def collect!
+        begin
+          @collector.call if @collector.is_a?(Proc)
+        rescue Exception => exc
+          log.exception(exc)
         end
       end
 
-      # The interval used to collect metrics. Either taken from the
-      # configuration file or overriden by subclasses.
-      def interval
-        @config.interval.tap{} or @collector.config.resolution
-      end
-
-      # This method must be implemented by subclasses to collect the desired
-      # metrics. This function is called every #interval seconds after the
-      # timer has been initialized by the #run method.
-      def collect!
-        raise NotImplementedError
+      # Loads a given ruby file, and runs instance_eval against it in the
+      # context of the current object.
+      #
+      # @param [String] filename  The absolute path to a plugin file.
+      # @return [Plugin] The instance for easy method chaining.
+      def from_file(filename)
+        if File.exists?(filename) && File.readable?(filename)
+          self.instance_eval(IO.read(filename), filename, 1)
+        else
+          raise IOError, "Cannot open or read #{filename}!"
+        end
+        self
       end
     end
   end

data/lib/ganymed/collectors/cpu.rb

@@ -0,0 +1,19 @@
+collect(0.3) do
+  return if not File.readable?('/proc/stat')
+
+  File.open('/proc/stat').each do |line|
+    next if not line =~ /^cpu /
+
+    cpu = line.chomp.split[1,7].map do |x|
+      x.to_i / 100
+    end
+
+    sampler.emit(:derive, "os.cpu.user", cpu[0])
+    sampler.emit(:derive, "os.cpu.nice", cpu[1])
+    sampler.emit(:derive, "os.cpu.system", cpu[2])
+    sampler.emit(:derive, "os.cpu.idle", cpu[3])
+    sampler.emit(:derive, "os.cpu.iowait", cpu[4])
+    sampler.emit(:derive, "os.cpu.irq", cpu[5])
+    sampler.emit(:derive, "os.cpu.softirq", cpu[6])
+  end
+end

data/lib/ganymed/collectors/disk.rb

@@ -0,0 +1,25 @@
+require 'sys/filesystem'
+
+collect do
+  Sys::Filesystem.mounts do |mount|
+    st = Sys::Filesystem.stat(mount.mount_point)
+    next if st.blocks == 0 or st.files == 0
+
+    case mp = mount.mount_point
+    when '/'
+      'rootfs'
+    when /^\/.+$/
+      mp[1..-1].tr('^[a-z][0-9]', '-')
+    else
+      'unknown'
+    end.tap do |name|
+      next if config.exclude.map {|e| Regexp.new(e).match(name)}.any?
+
+      block_pc = 1.0 - (st.blocks_free.to_f / st.blocks.to_f)
+      processor.event("os.disk.#{name}.blocks", block_pc)
+
+      files_pc = 1.0 - (st.files_free.to_f / st.files.to_f)
+      processor.event("os.disk.#{name}.files", block_pc)
+    end
+  end
+end

data/lib/ganymed/collectors/iostat.rb

@@ -0,0 +1,16 @@
+Struct.new("IOStat",
+           :major, :minor, :dev,
+           :rio, :rmerge, :rsect, :ruse,
+           :wio, :wmerge, :wsect, :wuse,
+           :running, :use, :aveq)
+
+collect(0.3) do
+  return if not File.readable?('/proc/diskstats')
+  File.open('/proc/diskstats').each do |line|
+    ios = Struct::IOStat.new(*line.strip.split(/\s+/))
+    next if config.skip_numbered.tap{} and ios.dev =~ /\d+$/
+    next if config.exclude.map {|e| Regexp.new(e).match(ios.dev)}.any?
+    sampler.emit(:derive, "os.iostat.#{ios.dev}.rsect", ios.rsect)
+    sampler.emit(:derive, "os.iostat.#{ios.dev}.wsect", ios.wsect)
+  end
+end

data/lib/ganymed/collectors/load.rb

@@ -0,0 +1,7 @@
+collect(5) do
+  return if not File.readable?('/proc/loadavg')
+  File.open('/proc/loadavg') do |f|
+    loadavg = f.read.chomp.split[0,3].map(&:to_f)
+    sampler.emit(:gauge, "os.loadavg", loadavg[0])
+  end
+end

data/lib/ganymed/collectors/memory.rb

@@ -0,0 +1,39 @@
+collect(5) do
+  return if not File.readable?('/proc/meminfo')
+
+  # calculate app memory from total
+  apps = 0
+
+  File.open('/proc/meminfo').each do |line|
+    key, value, unit = line.chomp.split
+    key.gsub!(/:/, '')
+    value = value.to_i * 1024
+
+    case key
+    when "MemTotal"
+      apps = value
+    when "MemFree"
+      sampler.emit(:gauge, "os.mem.unused", value)
+      apps -= value
+    when "Buffers"
+      sampler.emit(:gauge, "os.mem.buffers", value)
+      apps -= value
+    when "Cached"
+      sampler.emit(:gauge, "os.mem.cache", value)
+      apps -= value
+    when "SwapCached"
+      sampler.emit(:gauge, "os.mem.swap_cache", value)
+      apps -= value
+    when "Slab"
+      sampler.emit(:gauge, "os.mem.slab_cache", value)
+      apps -= value
+    when "PageTables"
+      sampler.emit(:gauge, "os.mem.page_tables", value)
+      apps -= value
+    when "Mapped"
+      sampler.emit(:gauge, "os.mem.mapped", value)
+    end
+  end
+
+  sampler.emit(:gauge, "os.mem.apps", apps)
+end

data/lib/ganymed/collectors/metadata.rb

@@ -0,0 +1,22 @@
+require 'ohai'
+
+collect do
+  Ohai::System.new.tap do |ohai|
+    %w(
+      os
+      kernel
+      hostname
+      keys
+      network
+      platform
+      uptime
+      virtualization
+    ).each do |plugin|
+      ohai.require_plugin(plugin)
+    end
+    processor.metadata(ohai.data)
+
+    # ohai doesn't cleanup after itself
+    true while Process.wait(-1, Process::WNOHANG) rescue nil
+  end
+end

data/lib/ganymed/collectors/network.rb

@@ -0,0 +1,26 @@
+# bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed
+Struct.new("NetworkStat", :dev,
+           :rbytes, :rpackets, :rerrs, :rdrop, :rfifo, :rframe, :rcompressed, :rmulticast,
+           :wbytes, :wpackets, :werrs, :wdrop, :wfifo, :wcolls, :wcarrier, :wcompressed)
+
+collect(0.3) do
+  return if not File.readable?('/proc/net/dev')
+
+  File.open('/proc/net/dev') do |f|
+    f.each do |line|
+      next unless line =~ /:/
+
+      ns = Struct::NetworkStat.new(*line.strip.split(/\s+/))
+      ns.dev.sub!(/:$/, '')
+
+      next if config.exclude.map {|e| Regexp.new(e).match(ns.dev)}.any?
+
+      sampler.emit(:derive, "os.net.#{ns.dev}.rbytes", ns.rbytes)
+      sampler.emit(:derive, "os.net.#{ns.dev}.wbytes", ns.wbytes)
+      sampler.emit(:derive, "os.net.#{ns.dev}.rpackets", ns.rpackets)
+      sampler.emit(:derive, "os.net.#{ns.dev}.wpackets", ns.wpackets)
+      sampler.emit(:derive, "os.net.#{ns.dev}.rerrs", ns.rerrs)
+      sampler.emit(:derive, "os.net.#{ns.dev}.werrs", ns.werrs)
+    end
+  end
+end

data/lib/ganymed/collectors/process.rb

@@ -0,0 +1,16 @@
+collect(0.3) do
+  return if not File.readable?('/proc/stat')
+  File.open('/proc/stat').each do |line|
+    key, value = line.chomp.split
+    case key
+    when "ctxt"
+      sampler.emit(:derive, "os.procs.switch", value.to_i)
+    when "processes"
+      sampler.emit(:derive, "os.procs.forks", value.to_i)
+    when "procs_running"
+      sampler.emit(:gauge, "os.procs.running", value.to_i)
+    when "procs_blocked"
+      sampler.emit(:gauge, "os.procs.blocked", value.to_i)
+    end
+  end
+end

data/lib/ganymed/collectors/uptime.rb

@@ -0,0 +1,8 @@
+collect do
+  return if not File.readable?('/proc/stat')
+  File.open('/proc/stat').each do |line|
+    next if not line =~ /^btime /
+    boottime = Time.at(line.chomp.split[1].to_i)
+    processor.event("os.reboot", 1, :now => boottime)
+  end
+end

data/lib/ganymed/config.yml

@@ -38,22 +38,18 @@ generic:
       port: 1337
 
   collectors:
-    - klass: Ganymed::Collector::Metadata
-    - klass: Ganymed::Collector::Uptime
-    - klass: Ganymed::Collector::Load
-    - klass: Ganymed::Collector::CPU
-    - klass: Ganymed::Collector::Procs
-    - klass: Ganymed::Collector::Network
+    network:
       exclude:
         - ^lo$
         - ^sit
-    - klass: Ganymed::Collector::Disk
+    disk:
       exclude:
         - ^dev$
         - ^dev-shm$
         - rc-init-d$
         - ^run$
         - ^sys-fs-cgroup$
-    - klass: Ganymed::Collector::IOStat
+        - ^var-tmp-metro
+    iostat:
       skip_numbered: true
       exclude: []

data/lib/ganymed/event.rb

@@ -5,11 +5,11 @@ module Ganymed
   #
   # n::  namespace (e.g. +os.cpu.idle+, +nginx.requests+, etc)
   # c::  a consolidation function used to build this event from multiple
-  #      samples
+  #      samples (_optional_)
   # o::  origin (e.g. +web1.example.com+, +myapplication+)
   # t::  timestamp as seconds since epoch
-  # r::  resolution of this event (i.e. how often is this event sent to the processor)
-  # v::  the value in integer or floating-point representation
+  # r::  resolution of this event (i.e. how often is this event sent to the processor, _optional_)
+  # v::  the event value (any object that can be serialized with BSON and JSON)
   #
   class Event
     # The namespace of this event.
@@ -17,7 +17,7 @@ module Ganymed
     attr_accessor :ns
 
     # The consolidation function used to build this event from multiple
-    # samples.
+    # samples if any.
     # @return [String]
     attr_accessor :cf
 
@@ -31,6 +31,9 @@ module Ganymed
 
     # The event resolution
     # @return [Fixnum]
+    #
+    # @note +nil+ or +0+ will bypass range checks for MongoDB, refer to the
+    #       {file:doc/configuration.md configuration} for details.
     attr_accessor :resolution
 
     # The actual value of this sample.
@@ -74,6 +77,8 @@ module Ganymed
     end
 
     # Validate this event.
+    #
+    # @return [void]
     def validate
       unless @ns =~ /^[a-z][a-z0-9_\.-]+[a-z0-9]$/
         raise "invalid namespace: #{@ns.inspect}"

data/lib/ganymed/master.rb

@@ -86,6 +86,7 @@ module Ganymed
 
       if config.profiling.perftools
         PerfTools::CpuProfiler.stop
+        `pprof.rb --svg profile > profile.svg`
       end
 
       if config.profiling.rubyprof

data/lib/ganymed/mongodb.rb

@@ -7,6 +7,7 @@ module Ganymed
 
     def initialize(config)
       @config = config
+      @collections = {}
       log.info("using MongoDB at #{config.host}:#{config.port}/#{config.database}")
     end
 
@@ -22,7 +23,7 @@ module Ganymed
     end
 
     def collection(ns)
-      db[ns].tap do |col|
+      @collections[ns] ||= db[ns].tap do |col|
         col.ensure_index([['c', ::Mongo::ASCENDING]])
         col.ensure_index([['o', ::Mongo::ASCENDING]])
         col.ensure_index([['t', ::Mongo::ASCENDING]])

data/lib/ganymed/processor.rb

@@ -1,18 +1,35 @@
 require 'eventmachine'
 require 'msgpack'
 
+require 'ganymed/client'
 require 'ganymed/event'
 require 'ganymed/mongodb'
 require 'ganymed/websocket'
 
 module Ganymed
+
+  ##
+  # The Processor accept {Event events} from a UDP socket and stores these
+  # metrics into MongoDB. All metrics are also published to interested parties
+  # via WebSocket.
+  #
   class Processor
     def initialize(config)
       @config = config
       @db = MongoDB.new(config.processor.mongodb)
       @websocket = Websocket.new(config.processor.websocket, @db)
       @client = Client.new(:sampler => @config.client.sampler)
-      listen!
+      cache! and listen!
+    end
+
+    def cache!
+      @metadata = Hash.new do |hsh, key|
+        hsh[key] = []
+      end
+
+      @db['metadata'].find({}, fields: [:_id, :namespaces]).each do |result|
+        @metadata[result['_id']] = result['namespaces']
+      end
     end
 
     def listen!
@@ -36,20 +53,19 @@ module Ganymed
     end
 
     def process_event(event)
-      # insert origin + namespace into metadata
-      metadata = @db['metadata'].find_and_modify({
-        query: {:_id => event.origin},
-        update: {:$addToSet => {:namespaces => event.ns}},
-        upsert: true,
-      })
+      if not @metadata[event.origin].include?(event.ns)
+        # insert origin + namespace into metadata
+        metadata = @db['metadata'].find_and_modify({
+          query: {:_id => event.origin},
+          update: {:$addToSet => {:namespaces => event.ns}},
+          upsert: true,
+        })
 
-      # set defaults in case find_and_modify returned an empty document
-      metadata['_id'] ||= event.origin
-      metadata['namespaces'] ||= []
+        # set defaults in case find_and_modify returned an empty document
+        metadata['_id'] ||= event.origin
+        metadata['namespaces'] ||= [event.ns]
 
-      # if metadata has changed notify websocket clients
-      if not metadata['namespaces'].include?(event.ns)
-        metadata['namespaces'] |= [event.ns]
+        # notify websocket clients
         @websocket.send(:metadata, [metadata])
       end
 
@@ -59,7 +75,7 @@ module Ganymed
       end
 
       # skip events that are produced below the mongodb storage threshold
-      return if (1..@config.resolution).include?(event.resolution)
+      return if (1...@config.resolution).include?(event.resolution)
 
       # insert event into ns collection
       @db.collection(event.ns).update({

data/lib/ganymed/sampler.rb

@@ -1,11 +1,10 @@
 require 'active_support/inflector'
 require 'eventmachine'
-require 'madvertise/ext/enumerable'
-require 'msgpack'
-require 'socket'
 require 'inline'
+require 'madvertise/ext/enumerable'
 
 require 'ganymed'
+require 'ganymed/client'
 
 module Ganymed
 

data/lib/ganymed/version.rb

@@ -1,3 +1,4 @@
 module Ganymed
-  VERSION = '0.1.2'
+  # @private
+  VERSION = '0.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ganymed
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-25 00:00:00.000000000 Z
+date: 2012-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &12783700 !ruby/object:Gem::Requirement
+  requirement: &19204520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '3.2'
   type: :runtime
   prerelease: false
-  version_requirements: *12783700
+  version_requirements: *19204520
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &12781700 !ruby/object:Gem::Requirement
+  requirement: &19202420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,21 @@ dependencies:
         version: 0.12.10
   type: :runtime
   prerelease: false
-  version_requirements: *12781700
+  version_requirements: *19202420
+- !ruby/object:Gem::Dependency
+  name: ganymed-client
+  requirement: &19227700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *19227700
 - !ruby/object:Gem::Dependency
   name: madvertise-ext
-  requirement: &12780400 !ruby/object:Gem::Requirement
+  requirement: &19226300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +54,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *12780400
+  version_requirements: *19226300
 - !ruby/object:Gem::Dependency
   name: madvertise-logging
-  requirement: &12806280 !ruby/object:Gem::Requirement
+  requirement: &19225180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +65,10 @@ dependencies:
         version: 0.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *12806280
+  version_requirements: *19225180
 - !ruby/object:Gem::Dependency
   name: mixlib-cli
-  requirement: &12805060 !ruby/object:Gem::Requirement
+  requirement: &19224340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *12805060
+  version_requirements: *19224340
 - !ruby/object:Gem::Dependency
-  name: servolux
-  requirement: &12803060 !ruby/object:Gem::Requirement
+  name: RubyInline
+  requirement: &19223140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *12803060
+  version_requirements: *19223140
 - !ruby/object:Gem::Dependency
-  name: RubyInline
-  requirement: &12800700 !ruby/object:Gem::Requirement
+  name: servolux
+  requirement: &19220580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *12800700
+  version_requirements: *19220580
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &12815620 !ruby/object:Gem::Requirement
+  requirement: &19234260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *12815620
+  version_requirements: *19234260
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &12812240 !ruby/object:Gem::Requirement
+  requirement: &19231900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +120,10 @@ dependencies:
         version: '1.6'
   type: :runtime
   prerelease: false
-  version_requirements: *12812240
+  version_requirements: *19231900
 - !ruby/object:Gem::Dependency
   name: em-websocket
-  requirement: &12810000 !ruby/object:Gem::Requirement
+  requirement: &19229040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +131,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *12810000
+  version_requirements: *19229040
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &12807700 !ruby/object:Gem::Requirement
+  requirement: &19257780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,10 +142,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *12807700
+  version_requirements: *19257780
 - !ruby/object:Gem::Dependency
   name: sys-filesystem
-  requirement: &12823980 !ruby/object:Gem::Requirement
+  requirement: &19255820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -142,10 +153,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *12823980
+  version_requirements: *19255820
 - !ruby/object:Gem::Dependency
   name: ohai
-  requirement: &12822240 !ruby/object:Gem::Requirement
+  requirement: &19252280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -153,7 +164,7 @@ dependencies:
         version: 0.6.12
   type: :runtime
   prerelease: false
-  version_requirements: *12822240
+  version_requirements: *19252280
 description: Ganymed is an event collection daemon
 email:
 - bb@xnull.de
@@ -175,16 +186,16 @@ files:
 - contrib/cpuhog.c
 - ganymed.gemspec
 - lib/ganymed.rb
-- lib/ganymed/client.rb
 - lib/ganymed/collector.rb
-- lib/ganymed/collector/cpu.rb
-- lib/ganymed/collector/disk.rb
-- lib/ganymed/collector/iostat.rb
-- lib/ganymed/collector/load.rb
-- lib/ganymed/collector/metadata.rb
-- lib/ganymed/collector/network.rb
-- lib/ganymed/collector/procs.rb
-- lib/ganymed/collector/uptime.rb
+- lib/ganymed/collectors/cpu.rb
+- lib/ganymed/collectors/disk.rb
+- lib/ganymed/collectors/iostat.rb
+- lib/ganymed/collectors/load.rb
+- lib/ganymed/collectors/memory.rb
+- lib/ganymed/collectors/metadata.rb
+- lib/ganymed/collectors/network.rb
+- lib/ganymed/collectors/process.rb
+- lib/ganymed/collectors/uptime.rb
 - lib/ganymed/config.yml
 - lib/ganymed/event.rb
 - lib/ganymed/master.rb
@@ -228,7 +239,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -813213877420202571
+      hash: -4122495108718523153
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -237,7 +248,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -813213877420202571
+      hash: -4122495108718523153
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.17

data/lib/ganymed/client.rb

@@ -1,132 +0,0 @@
-require 'active_support/core_ext/array/extract_options'
-require 'socket'
-require 'msgpack'
-
-module Ganymed
-
-  ##
-  # The Client connects to the {Sampler} and/or {Processor} and emits samples
-  # and/or {Event events} respectively.
-  #
-  class Client
-
-    # The {Processor} {ProcessorSocket socket}.
-    attr_reader :processor
-
-    # The {Sampler} {SamplerSocket socket}.
-    attr_reader :sampler
-
-    # Create a new client instance.
-    #
-    # @param [Hash] opts  Client options.
-    # @option opts [Hash] :processor  Options passed to the {ProcessorSocket}.
-    # @option opts [Hash] :sampler  Options passed to the {SamplerSocket}.
-    def initialize(opts)
-      @processor = ProcessorSocket.new(opts[:processor]) if opts[:processor]
-      @sampler = SamplerSocket.new(opts[:sampler]) if opts[:sampler]
-    end
-
-    ##
-    # A simple UDPSocket wrapper.
-    #
-    class Socket
-
-      # Create a new Socket instance.
-      #
-      # @param [Hash] opts  Socket options.
-      # @option opts [String] :host  Host to connect to.
-      # @option opts [Fixnum] :port  Port to connect to.
-      # @option opts [String] :origin Origin of Samples/Events. Defaults to the
-      #                               fully-qualified hostname.
-      def initialize(opts)
-        @host, @port = opts[:host], opts[:port]
-        @socket = UDPSocket.new
-        @origin = opts[:origin] || ::Socket.gethostbyname(::Socket.gethostname).first
-      end
-
-      # Send data to the socket.
-      #
-      # @param [String] data  The data to send.
-      # @param [Fixnum] flags  Socket flags.
-      def send(data, flags=0)
-        @socket.send(data, flags, @host, @port)
-      end
-    end
-
-    ##
-    # A {Socket} that emits samples to a {Sampler}.
-    #
-    class SamplerSocket < Socket
-
-      # Emit a new sample.
-      #
-      # @param [String, Symbol] ds  Sample data source.
-      # @param [String] ns  {Event} namespace.
-      # @param [Fixnum, Float] value  Sample value.
-      # @param [Time] now  Sample timestamp.
-      def emit(ds, ns, value, now=Time.now.utc)
-        data = [ds.to_s, ns, @origin, now.to_f, value.to_f]
-        send(data.pack("Z*Z*Z*GG"))
-      end
-    end
-
-    ##
-    # A {Socket} that emits {Event events} to a {Processor}.
-    #
-    class ProcessorSocket < Socket
-
-      # Emit a new {Event}.
-      #
-      # @param [String] ns  {Event} namespace.
-      # @param [Object] value  {Event} value.
-      # @param [Hash] opts  Options
-      # @option opts [String] cf  Consolidation function used in this event.
-      # @option opts [Time] now  {Event} timestamp.
-      # @option opts [Fixnum] resolution  {Event} resolution.
-      # @option opts [String] origin  {Event} origin.
-      def event(ns, value, opts={})
-        opts = {
-          :cf => nil,
-          :now => Time.now.utc,
-          :resolution => 0,
-          :origin => @origin,
-        }.merge(opts)
-
-        {
-          :_type => :event,
-          :n => ns,
-          :c => opts[:cf],
-          :o => opts[:origin],
-          :t => opts[:now].to_i,
-          :r => opts[:resolution].to_i,
-          :v => value
-        }.tap do |data|
-          send(data.to_msgpack)
-        end
-      end
-
-      # Emit a metadata object to the {Processor}.
-      #
-      # A metadata object contains arbitrary data related to the origin. This
-      # can be queried by {Websocket} clients for displaying information about
-      # origins.
-      #
-      # @param [Hash] data  Metadata object.
-      # @param [Hash] opts  Options
-      # @option opts [String] origin  Metadata origin. Defaults to the FQDN.
-      def metadata(data, opts={})
-        opts = {
-          :origin => @origin,
-        }.merge(opts)
-
-        {
-          :_type => :metadata,
-          :origin => opts[:origin],
-          :data => data,
-        }.tap do |data|
-          send(data.to_msgpack)
-        end
-      end
-    end
-  end
-end

data/lib/ganymed/collector/cpu.rb

@@ -1,32 +0,0 @@
-require 'ganymed/collector'
-
-module Ganymed
-  class Collector
-    class CPU < Base
-      def collect!
-        return if not File.readable?('/proc/stat')
-        File.open('/proc/stat').each do |line|
-          next if not line =~ /^cpu /
-          cpu = line.chomp.split[1,7].map do |x|
-            x.to_i / hz
-          end
-          @sampler.emit(:derive, "os.cpu.user", cpu[0])
-          @sampler.emit(:derive, "os.cpu.nice", cpu[1])
-          @sampler.emit(:derive, "os.cpu.system", cpu[2])
-          @sampler.emit(:derive, "os.cpu.idle", cpu[3])
-          @sampler.emit(:derive, "os.cpu.iowait", cpu[4])
-          @sampler.emit(:derive, "os.cpu.irq", cpu[5])
-          @sampler.emit(:derive, "os.cpu.softirq", cpu[6])
-        end
-      end
-
-      def interval
-        @config.interval.tap{} or 0.2
-      end
-
-      def hz
-        @config.hz.tap{} or 100
-      end
-    end
-  end
-end

data/lib/ganymed/collector/disk.rb

@@ -1,36 +0,0 @@
-require 'sys/filesystem'
-
-module Ganymed
-  class Collector
-    class Disk < Base
-      def collect!
-        Sys::Filesystem.mounts do |mount|
-          st = Sys::Filesystem.stat(mount.mount_point)
-          next if st.blocks == 0 or st.files == 0
-
-          name = name(mount.mount_point)
-          next if @config.exclude.map {|e| Regexp.new(e).match(name)}.any?
-
-          block_pc = 1.0 - (st.blocks_free.to_f / st.blocks.to_f)
-          @processor.event("os.disk.#{name}.blocks", block_pc)
-
-          files_pc = 1.0 - (st.files_free.to_f / st.files.to_f)
-          @processor.event("os.disk.#{name}.files", block_pc)
-        end
-      end
-
-      private
-
-      def name(mount)
-        case mount
-        when '/'
-          'rootfs'
-        when /^\/.+$/
-          mount[1..-1].tr('^[a-z][0-9]', '-')
-        else
-          'unknown'
-        end
-      end
-    end
-  end
-end

data/lib/ganymed/collector/iostat.rb

@@ -1,26 +0,0 @@
-Struct.new("IOStat",
-           :major, :minor, :dev,
-           :rio, :rmerge, :rsect, :ruse,
-           :wio, :wmerge, :wsect, :wuse,
-           :running, :use, :aveq)
-
-module Ganymed
-  class Collector
-    class IOStat < Base
-      def collect!
-        return if not File.readable?('/proc/diskstats')
-        File.open('/proc/diskstats').each do |line|
-          ios = Struct::IOStat.new(*line.strip.split(/\s+/))
-          next if @config.skip_numbered.tap{} and ios.dev =~ /\d+$/
-          next if @config.exclude.map {|e| Regexp.new(e).match(ios.dev)}.any?
-          @sampler.emit(:derive, "os.iostat.#{ios.dev}.rsect", ios.rsect)
-          @sampler.emit(:derive, "os.iostat.#{ios.dev}.wsect", ios.wsect)
-        end
-      end
-
-      def interval
-        @config.interval.tap{} or 0.2
-      end
-    end
-  end
-end

data/lib/ganymed/collector/load.rb

@@ -1,19 +0,0 @@
-require 'ganymed/collector'
-
-module Ganymed
-  class Collector
-    class Load < Base
-      def collect!
-        return if not File.readable?('/proc/loadavg')
-        File.open('/proc/loadavg') do |f|
-          loadavg = f.read.chomp.split[0,3].map(&:to_f)
-          @sampler.emit(:gauge, "os.loadavg", loadavg[0])
-        end
-      end
-
-      def interval
-        @config.interval.tap{} or 0.2
-      end
-    end
-  end
-end

data/lib/ganymed/collector/metadata.rb

@@ -1,34 +0,0 @@
-require 'ohai'
-
-require 'ganymed/collector'
-
-module Ganymed
-  class Collector
-    class Metadata < Base
-      def collect!
-        Ohai::System.new.tap do |ohai|
-          %w(
-            os
-            kernel
-            hostname
-            keys
-            network
-            platform
-            uptime
-            virtualization
-          ).each do |plugin|
-            ohai.require_plugin(plugin)
-          end
-          @processor.metadata(ohai.data)
-
-          # ohai doesn't cleanup after itself
-          true while Process.wait(-1, Process::WNOHANG) rescue nil
-        end
-      end
-
-      def interval
-        nil
-      end
-    end
-  end
-end

data/lib/ganymed/collector/network.rb

@@ -1,36 +0,0 @@
-# bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed
-Struct.new("NetworkStat", :dev,
-           :rbytes, :rpackets, :rerrs, :rdrop, :rfifo, :rframe, :rcompressed, :rmulticast,
-           :wbytes, :wpackets, :werrs, :wdrop, :wfifo, :wcolls, :wcarrier, :wcompressed)
-
-module Ganymed
-  class Collector
-    class Network < Base
-      def collect!
-        return if not File.readable?('/proc/net/dev')
-
-        File.open('/proc/net/dev') do |f|
-          f.each do |line|
-            next unless line =~ /:/
-
-            ns = Struct::NetworkStat.new(*line.strip.split(/\s+/))
-            ns.dev.sub!(/:$/, '')
-
-            next if @config.exclude.map {|e| Regexp.new(e).match(ns.dev)}.any?
-
-            @sampler.emit(:derive, "os.net.#{ns.dev}.rbytes", ns.rbytes)
-            @sampler.emit(:derive, "os.net.#{ns.dev}.wbytes", ns.wbytes)
-            @sampler.emit(:derive, "os.net.#{ns.dev}.rpackets", ns.rpackets)
-            @sampler.emit(:derive, "os.net.#{ns.dev}.wpackets", ns.wpackets)
-            @sampler.emit(:derive, "os.net.#{ns.dev}.rerrs", ns.rerrs)
-            @sampler.emit(:derive, "os.net.#{ns.dev}.werrs", ns.werrs)
-          end
-        end
-      end
-
-      def interval
-        @config.interval.tap{} or 0.1
-      end
-    end
-  end
-end

data/lib/ganymed/collector/procs.rb

@@ -1,22 +0,0 @@
-module Ganymed
-  class Collector
-    class Procs < Base
-      def collect!
-        return if not File.readable?('/proc/stat')
-        File.open('/proc/stat').each do |line|
-          next if not line =~ /^procs_/
-          key, value = line.chomp.split
-          if key == "procs_running"
-            @sampler.emit(:gauge, "os.procs.running", value.to_i)
-          elsif key == "procs_blocked"
-            @sampler.emit(:gauge, "os.procs.blocked", value.to_i)
-          end
-        end
-      end
-
-      def interval
-        @config.interval.tap{} or 0.2
-      end
-    end
-  end
-end

data/lib/ganymed/collector/uptime.rb

@@ -1,16 +0,0 @@
-require 'ganymed/collector'
-
-module Ganymed
-  class Collector
-    class Uptime < Base
-      def collect!
-        return if not File.readable?('/proc/stat')
-        File.open('/proc/stat').each do |line|
-          next if not line =~ /^btime /
-          boottime = Time.at(line.chomp.split[1].to_i)
-          @processor.event("os.reboot", 1, :now => boottime)
-        end
-      end
-    end
-  end
-end