ganymed 0.1.0

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage/
+doc/
+ganymed.pid
+ganymed.profile*
+lib/bundler/man
+node_modules/
+old/
+pkg
+rdoc
+spec/reports
+test.js
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper --color --format documentation

data/.rvmrc

@@ -0,0 +1 @@
+rvm use --create ruby-1.9.3-p125@ganymed

data/.yardopts

@@ -0,0 +1 @@
+--protected --no-private

data/Gemfile

@@ -0,0 +1,20 @@
+source :rubygems
+
+gemspec
+
+group :development, :test do
+  gem 'bson_ext'
+  gem 'bundler'
+  gem 'debugger'
+  gem 'perftools.rb'
+  gem 'pry'
+  gem 'pry-doc'
+  gem 'redcarpet'
+  gem 'reek'
+  gem 'rspec'
+  gem 'ruby-prof'
+  gem 'ruby2ruby', '=1.3.0' # 1.3.1 is broken :(
+  gem 'simplecov'
+  gem 'syslogger'
+  gem 'yard'
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Benedikt Böhm
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# Ganymed
+
+Ganymed is an event collection daemon.
+
+Ganymed processes system and application metrics in realtime and can easily
+handle thousands of samples per second.
+
+* the *sampler* accepts high frequency samples. these samples are then
+  consolidated based on the data source type and emitted to the *processor*.
+
+* the *processor* accepts low frequency data points (possibly created by the
+  *sampler*) and publishes these to interested parties via WebSocket in
+  realtime. All samples are also stored in MongoDB for later analysis.
+
+* the *collector* will send various system metrics (cpu, memory, network, etc)
+  to the *sampler* and *processor*
+
+## Installation
+
+TBD
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+#!/usr/bin/env rake
+
+require "bundler/setup"
+require "bundler/gem_tasks"
+
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+task :default => [:spec]

data/bin/ganymed

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$:.unshift(File.join(File.expand_path("../..", __FILE__), 'lib'))
+
+require 'ganymed/master'
+cli = Ganymed::Master::CLI.parse_options
+server = Ganymed::Master.new(cli.config)
+
+if cli.config[:daemonize] or cli.config[:kill]
+  daemon = Servolux::Daemon.new(:server => server)
+
+  if cli.config[:kill]
+    daemon.shutdown
+  else
+    daemon.startup
+  end
+else
+  server.startup
+end

data/ganymed.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "ganymed/version"
+
+Gem::Specification.new do |s|
+  s.name = "ganymed"
+  s.version = Ganymed::VERSION
+  s.authors = ["Benedikt Böhm"]
+  s.email = ["bb@xnull.de"]
+  s.homepage = "http://github.com/zenops/ganymed"
+  s.summary = %q{Ganymed is an event collection daemon}
+  s.description = %q{Ganymed is an event collection daemon}
+
+  s.files            = `git ls-files`.split("\n")
+  s.test_files       = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables      = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths    = ["lib"]
+
+  # master
+  s.add_dependency "activesupport", ">= 3.2"
+  s.add_dependency "eventmachine", ">= 0.12.10"
+  s.add_dependency "madvertise-ext", ">= 0.1.2"
+  s.add_dependency "madvertise-logging", ">= 0.3.2"
+  s.add_dependency "mixlib-cli"
+  s.add_dependency "servolux"
+
+  # processor
+  s.add_dependency "msgpack"
+  s.add_dependency "mongo", ">= 1.6"
+
+  # websocket
+  s.add_dependency "em-websocket"
+  s.add_dependency "yajl-ruby"
+
+  # collector
+  s.add_dependency "sys-filesystem"
+  s.add_dependency "ohai", ">= 0.6.12"
+end

data/lib/ganymed/client.rb

@@ -0,0 +1,132 @@
+require 'active_support/core_ext/array/extract_options'
+require 'socket'
+require 'msgpack'
+
+module Ganymed
+
+  ##
+  # The Client connects to the {Sampler} and/or {Processor} and emits samples
+  # and/or {Event events} respectively.
+  #
+  class Client
+
+    # The {Processor} {ProcessorSocket socket}.
+    attr_reader :processor
+
+    # The {Sampler} {SamplerSocket socket}.
+    attr_reader :sampler
+
+    # Create a new client instance.
+    #
+    # @param [Hash] opts  Client options.
+    # @option opts [Hash] :processor  Options passed to the {ProcessorSocket}.
+    # @option opts [Hash] :sampler  Options passed to the {SamplerSocket}.
+    def initialize(opts)
+      @processor = ProcessorSocket.new(opts[:processor]) if opts[:processor]
+      @sampler = SamplerSocket.new(opts[:sampler]) if opts[:sampler]
+    end
+
+    ##
+    # A simple UDPSocket wrapper.
+    #
+    class Socket
+
+      # Create a new Socket instance.
+      #
+      # @param [Hash] opts  Socket options.
+      # @option opts [String] :host  Host to connect to.
+      # @option opts [Fixnum] :port  Port to connect to.
+      # @option opts [String] :origin Origin of Samples/Events. Defaults to the
+      #                               fully-qualified hostname.
+      def initialize(opts)
+        @host, @port = opts[:host], opts[:port]
+        @socket = UDPSocket.new
+        @origin = opts[:origin] || ::Socket.gethostbyname(::Socket.gethostname).first
+      end
+
+      # Send data to the socket.
+      #
+      # @param [String] data  The data to send.
+      # @param [Fixnum] flags  Socket flags.
+      def send(data, flags=0)
+        @socket.send(data, flags, @host, @port)
+      end
+    end
+
+    ##
+    # A {Socket} that emits samples to a {Sampler}.
+    #
+    class SamplerSocket < Socket
+
+      # Emit a new sample.
+      #
+      # @param [String, Symbol] ds  Sample data source.
+      # @param [String] ns  {Event} namespace.
+      # @param [Fixnum, Float] value  Sample value.
+      # @param [Time] now  Sample timestamp.
+      def emit(ds, ns, value, now=Time.now.utc)
+        data = [ds.to_s, ns, @origin, now.to_i + (now.usec * 1e-6), value.to_f]
+        send(data.pack("Z*Z*Z*GG"))
+      end
+    end
+
+    ##
+    # A {Socket} that emits {Event events} to a {Processor}.
+    #
+    class ProcessorSocket < Socket
+
+      # Emit a new {Event}.
+      #
+      # @param [String] ns  {Event} namespace.
+      # @param [Fixnum, Float] value  {Event} value.
+      # @param [Hash] opts  Options
+      # @option opts [String] cf  Consolidation function used in this event.
+      # @option opts [Time] now  {Event} timestamp.
+      # @option opts [String] origin  {Event} origin.
+      # @option opts [Array] modifiers  Event modifiers.
+      def event(ns, value, opts={})
+        opts = {
+          :cf => nil,
+          :now => Time.now.utc,
+          :origin => @origin,
+          :modifiers => [],
+        }.merge(opts)
+
+        {
+          :_type => :event,
+          :n => ns,
+          :m => opts[:modifiers],
+          :c => opts[:cf],
+          :o => opts[:origin],
+          :t => opts[:now].to_i,
+          :v => value
+        }.tap do |data|
+          send(data.to_msgpack)
+        end
+      end
+
+      # Emit a metadata object to the {Processor}.
+      #
+      # A metadata object contains arbitrary data related to the origin. This
+      # can be queried by {Websocket} clients for displaying information about
+      # origins.
+      #
+      # @param [Hash] data  Metadata object.
+      # @param [Hash] opts  Options
+      # @option opts [String] origin  Metadata origin. Defaults to the FQDN.
+      def metadata(data, opts={})
+        opts = {
+          :origin => @origin,
+        }.merge(opts)
+
+        {
+          :_type => :metadata,
+          :origin => opts[:origin],
+          :data => data,
+        }.tap do |data|
+          send(data.to_msgpack)
+        end
+      end
+    end
+  end
+end

data/lib/ganymed/collector/cpu.rb

@@ -0,0 +1,30 @@
+require 'ganymed/collector'
+
+module Ganymed
+  class Collector
+    class CPU < Base
+      def collect!
+        File.open('/proc/stat') do |f|
+          cpu = f.readline.chomp.split[1,7].map do |x|
+            x.to_i / hz
+          end
+          @sampler.emit(:derive, "os.cpu.user", cpu[0])
+          @sampler.emit(:derive, "os.cpu.nice", cpu[1])
+          @sampler.emit(:derive, "os.cpu.system", cpu[2])
+          @sampler.emit(:derive, "os.cpu.idle", cpu[3])
+          @sampler.emit(:derive, "os.cpu.iowait", cpu[4])
+          @sampler.emit(:derive, "os.cpu.irq", cpu[5])
+          @sampler.emit(:derive, "os.cpu.softirq", cpu[6])
+        end
+      end
+
+      def interval
+        @config.interval.tap{} or 0.2
+      end
+
+      def hz
+        @config.hz.tap{} or 100
+      end
+    end
+  end
+end

data/lib/ganymed/collector/disk.rb

@@ -0,0 +1,35 @@
+require 'sys/filesystem'
+
+module Ganymed
+  class Collector
+    class Disk < Base
+      def collect!
+        Sys::Filesystem.mounts do |mount|
+          st = Sys::Filesystem.stat(mount.mount_point)
+          next if st.blocks == 0 or st.files == 0
+
+          name = name(mount.mount_point)
+
+          block_pc = 1.0 - (st.blocks_free.to_f / st.blocks.to_f)
+          @processor.event("os.disk.#{name}.blocks", block_pc)
+
+          files_pc = 1.0 - (st.files_free.to_f / st.files.to_f)
+          @processor.event("os.disk.#{name}.files", block_pc)
+        end
+      end
+
+      private
+
+      def name(mount)
+        case mount
+        when '/'
+          'rootfs'
+        when /^\/.+$/
+          mount[1..-1].tr('^[a-z][0-9]', '-')
+        else
+          'unknown'
+        end
+      end
+    end
+  end
+end

data/lib/ganymed/collector/iostat.rb

@@ -0,0 +1,25 @@
+Struct.new("IOStat",
+           :major, :minor, :dev,
+           :rio, :rmerge, :rsect, :ruse,
+           :wio, :wmerge, :wsect, :wuse,
+           :running, :use, :aveq)
+
+module Ganymed
+  class Collector
+    class IOStat < Base
+      def collect!
+        File.open('/proc/diskstats') do |f|
+          f.each do |line|
+            ios = Struct::IOStat.new(*line.strip.split(/\s+/))
+            @sampler.emit(:derive, "os.iostat.#{ios.dev}.rsect", ios.rsect)
+            @sampler.emit(:derive, "os.iostat.#{ios.dev}.wsect", ios.wsect)
+          end
+        end
+      end
+
+      def interval
+        @config.interval.tap{} or 0.2
+      end
+    end
+  end
+end

data/lib/ganymed/collector/load.rb

@@ -0,0 +1,18 @@
+require 'ganymed/collector'
+
+module Ganymed
+  class Collector
+    class Load < Base
+      def collect!
+        File.open('/proc/loadavg') do |f|
+          loadavg = f.read.chomp.split[0,3].map(&:to_f)
+          @sampler.emit(:gauge, "os.loadavg", loadavg[0])
+        end
+      end
+
+      def interval
+        @config.interval.tap{} or 0.2
+      end
+    end
+  end
+end

data/lib/ganymed/collector/metadata.rb

@@ -0,0 +1,31 @@
+require 'ohai'
+
+require 'ganymed/collector'
+
+module Ganymed
+  class Collector
+    class Metadata < Base
+      def collect!
+        Ohai::System.new.tap do |ohai|
+          %w(
+            os
+            kernel
+            hostname
+            keys
+            network
+            platform
+            uptime
+            virtualization
+          ).each do |plugin|
+            ohai.require_plugin(plugin)
+          end
+          @processor.metadata(ohai.data)
+        end
+      end
+
+      def interval
+        nil
+      end
+    end
+  end
+end

data/lib/ganymed/collector/network.rb

@@ -0,0 +1,30 @@
+# bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed
+Struct.new("NetworkStat", :dev,
+           :rbytes, :rpackets, :rerrs, :rdrop, :rfifo, :rframe, :rcompressed, :rmulticast,
+           :wbytes, :wpackets, :werrs, :wdrop, :wfifo, :wcolls, :wcarrier, :wcompressed)
+
+module Ganymed
+  class Collector
+    class Network < Base
+      def collect!
+        File.open('/proc/net/dev') do |f|
+          f.each do |line|
+            next unless line =~ /:/
+            ns = Struct::NetworkStat.new(*line.strip.split(/\s+/))
+            ns.dev.sub!(/:$/, '')
+            @sampler.emit(:derive, "os.net.#{ns.dev}.rbytes", ns.rbytes)
+            @sampler.emit(:derive, "os.net.#{ns.dev}.wbytes", ns.wbytes)
+            @sampler.emit(:derive, "os.net.#{ns.dev}.rpackets", ns.rpackets)
+            @sampler.emit(:derive, "os.net.#{ns.dev}.wpackets", ns.wpackets)
+            @sampler.emit(:derive, "os.net.#{ns.dev}.rerrs", ns.rerrs)
+            @sampler.emit(:derive, "os.net.#{ns.dev}.werrs", ns.werrs)
+          end
+        end
+      end
+
+      def interval
+        @config.interval.tap{} or 0.1
+      end
+    end
+  end
+end

data/lib/ganymed/collector/uptime.rb

@@ -0,0 +1,15 @@
+require 'ganymed/collector'
+
+module Ganymed
+  class Collector
+    class Uptime < Base
+      def collect!
+        File.open('/proc/uptime') do |f|
+          uptime = f.read.chomp.split.first.to_f
+          boottime = Time.now - uptime
+          @processor.event("os.reboot", 1, :now => boottime)
+        end
+      end
+    end
+  end
+end

data/lib/ganymed/collector.rb

@@ -0,0 +1,89 @@
+require 'eventmachine'
+
+require 'ganymed'
+require 'ganymed/client'
+
+module Ganymed
+
+  ##
+  # The Collector polls various system-level metrics that are not pushed to the
+  # {Processor} automatically.
+  #
+  class Collector
+    # The configuration object.
+    attr_reader :config
+
+    # The {Client} object.
+    attr_reader :client
+
+    # Create a new collector instance and initialize all configured collectors.
+    #
+    # @param [Configuration] config  The configuration object.
+    def initialize(config)
+      @config = config
+      @client = Client.new(:processor => @config.collector.processor,
+                           :sampler => @config.collector.sampler)
+
+      @config.collectors.each do |collector|
+        log.info("initializing collector #{collector.klass.demodulize}")
+        collector.klass.constantize.new(collector, self).run
+      end
+    end
+
+    ##
+    # A base class for collectors.
+    #
+    class Base
+
+      # Create a new data collector instance.
+      #
+      # @param [Configuration] config  The configuration object.
+      # @param [Collector] collector  The collector instance.
+      def initialize(config, collector)
+        @config, @collector = config, collector
+        @processor = @collector.client.processor
+        @sampler = @collector.client.sampler
+      end
+
+      # Start the EventMachine timer to collect metrics at the specified
+      # interval.
+      def run
+        timer do
+          begin
+            collect!
+          rescue Exception => e
+            log.exception(e)
+          end
+        end
+      end
+
+      # @private
+      def timer
+        if interval
+          EM.add_periodic_timer(interval) do
+            EM.defer { yield }
+          end
+        else
+          yield
+        end
+      end
+
+      # The interval used to collect metrics. Either taken from the
+      # configuration file or overriden by subclasses.
+      def interval
+        @config.interval.tap{} or @collector.config.resolution
+      end
+
+      # This method must be implemented by subclasses to collect the desired
+      # metrics. This function is called every #interval seconds after the
+      # timer has been initialized by the #run method.
+      def collect!
+        raise NotImplementedError
+      end
+    end
+  end
+end
+
+Dir[File.join(Ganymed::LIB_DIR, 'ganymed/collector/*.rb')].each do |f|
+  require f
+end

data/lib/ganymed/config.yml

@@ -0,0 +1,49 @@
+generic:
+  pid_file: /tmp/ganymed.pid
+
+  resolution: 300
+
+  profiling:
+    perftools: false
+    gcprofiler: false
+    rubyprof: false
+
+  processor:
+    enabled: true
+    listen:
+      host: 0.0.0.0
+      port: 1336
+    mongodb:
+      host: localhost
+      port: 27017
+      database: ganymed
+    websocket:
+      host: 0.0.0.0
+      port: 1338
+
+  sampler:
+    enabled: true
+    listen:
+      host: 127.0.0.1
+      port: 1337
+    emit:
+      host: 127.0.0.1
+      port: 1336
+
+  collector:
+    enabled: true
+    processor:
+      host: 127.0.0.1
+      port: 1336
+    sampler:
+      host: 127.0.0.1
+      port: 1337
+
+  collectors:
+    - klass: Ganymed::Collector::Metadata
+    - klass: Ganymed::Collector::Uptime
+    - klass: Ganymed::Collector::Load
+    - klass: Ganymed::Collector::CPU
+    - klass: Ganymed::Collector::Network
+    - klass: Ganymed::Collector::Disk
+    - klass: Ganymed::Collector::IOStat