ganymed 0.3.4 → 0.4.0

data/Gemfile

@@ -3,9 +3,8 @@ source :rubygems
 gemspec
 
 group :development, :test do
-  #gem 'ganymed-client', :path => '../ganymed-client'
-  gem 'bson_ext'
   gem 'bundler'
+  gem 'cane'
   gem 'debugger'
   gem 'perftools.rb'
   gem 'pry'

data/README.md

@@ -1,20 +1,12 @@
 # Ganymed
 
-Ganymed is an event collection daemon.
+Ganymed is a daemon that collects Metriks from third-party applications.
 
-Ganymed processes system and application metrics and events in realtime and can
-easily handle thousands of samples per second. Ganymed provides these services
-to administrators and third-party applications:
-
-* The *Sampler* accepts high frequency samples. these samples are then
-  consolidated based on the data source type and emitted to the *Processor*.
-
-* The *Processor* accepts metrics (events) and stores these metrics into
-  MongoDB. All metrics are also published to interested parties via WebSocket.
-
-* The *Collector* is a simple thread that polls various system-level metrics
-  that are not pushed to the *Sampler*/*Processor* from third-party
-  applications.
+The metriks gem and its logfile reporter are a great tool for rapid metric
+collection. However applications have to collect metrics themselves. In order
+to collect system-level metrics and metrics from applications like MySQL or
+MongoDB, these applications have to be polled from the outside. Ganymed polls
+thos applications and system-level metrics (in /proc or similar).
 
 ## Installation
 

data/ganymed.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |s|
   s.authors = ["Benedikt Böhm"]
   s.email = ["bb@xnull.de"]
   s.homepage = "http://github.com/zenops/ganymed"
-  s.summary = %q{Ganymed is an event collection daemon}
-  s.description = %q{Ganymed is an event collection daemon}
+  s.summary = %q{Ganymed is a daemon that collects Metriks from third-party applications}
+  s.description = %q{Ganymed is a daemon that collects Metriks from third-party applications}
 
   s.files            = `git ls-files`.split("\n")
   s.test_files       = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -19,27 +19,14 @@ Gem::Specification.new do |s|
   # master
   s.add_dependency "activesupport", ">= 3.2"
   s.add_dependency "eventmachine", ">= 0.12.10"
-  s.add_dependency "ganymed-client", ">= 0.2.0"
   s.add_dependency "madvertise-ext", ">= 0.1.2"
   s.add_dependency "madvertise-logging", ">= 0.3.2"
   s.add_dependency "mixlib-cli"
-  s.add_dependency "RubyInline"
   s.add_dependency "servolux"
 
-  # processor
-  s.add_dependency "msgpack"
-  s.add_dependency "mongo", ">= 1.6"
-
-  # websocket
-  s.add_dependency "em-websocket"
-  s.add_dependency "yajl-ruby"
-
   # collector
+  s.add_dependency "metriks"
+  s.add_dependency "metriks-derive"
   s.add_dependency "sys-filesystem"
   s.add_dependency "ohai", ">= 0.6.12"
-
-  # console
-  s.add_dependency "ripl"
-  s.add_dependency "ascii_charts"
-  s.add_dependency "terminal-table"
 end

data/lib/ganymed/collector.rb

@@ -1,13 +1,13 @@
 require 'eventmachine'
-
 require 'ganymed'
-require 'ganymed/client'
+require 'metriks'
+require 'metriks-derive'
 
 module Ganymed
 
   ##
   # The Collector is a simple thread that polls various system-level metrics
-  # that are not pushed to the {Sampler}/{Processor} from third-party
+  # that are not pushed to the {Processor} from third-party
   # applications.
   #
   # The Collector features a very simple DSL to implement custom collector
@@ -40,26 +40,11 @@ module Ganymed
     # @return [Section]
     attr_reader :config
 
-    # The client object.
-    # @return [Ganymed::Client]
-    attr_reader :client
-
     # Create a new collector instance and initialize all configured collectors.
     #
     # @param [Section] config  The configuration object.
     def initialize(config)
       @config = config
-
-      @config.client.sampler.tap do |sampler|
-        log.info("emitting collected samples to tcp##{sampler.host}:#{sampler.port}")
-        @sampler = Ganymed::Client::Sampler.connect(sampler.host, sampler.port)
-      end
-
-      @config.client.processor.tap do |processor|
-        log.info("emitting collected events to tcp##{processor.host}:#{processor.port}")
-        @processor = Ganymed::Client::Processor.connect(processor.host, processor.port)
-      end
-
       load_collectors
     end
 
@@ -70,7 +55,7 @@ module Ganymed
         config = @config.collectors[name.to_sym] || Section.new
 
         log.debug("loading collector #{name} from #{file}")
-        Plugin.new(config, @sampler, @processor).from_file(file).tap do |collector|
+        Plugin.new(config).from_file(file).tap do |collector|
           log.info("collecting #{name} metrics every #{collector.interval} seconds")
           collector.run
         end
@@ -105,20 +90,11 @@ module Ganymed
       # @return [Fixnum,Float]
       attr_accessor :interval
 
-      # Processor socket.
-      # @return [Ganymed::Client::Processor]
-      attr_accessor :processor
-
-      # Sampler socket.
-      # @return [Ganymed::Client::Sampler]
-      attr_accessor :sampler
-
       # Create a new plugin instance.
       #
       # @param [Section] config  The configuration object.
-      # @param [Ganymed::Client] client  A client instance.
-      def initialize(config, sampler, processor)
-        @config, @sampler, @processor = config, sampler, processor
+      def initialize(config)
+        @config = config
       end
 
       # Set the block used to collect metrics with this plugin.

data/lib/ganymed/collectors/cpu.rb

@@ -1,7 +1,7 @@
 collect do
   next if not File.readable?('/proc/stat')
 
-  File.open('/proc/stat') do | file|
+  File.open('/proc/stat') do |file|
     file.each do |line|
       next if not line =~ /^cpu /
 
@@ -9,13 +9,13 @@ collect do
         x.to_i / 100
       end
 
-      sampler.emit(:derive, "os.cpu.user", cpu[0])
-      sampler.emit(:derive, "os.cpu.nice", cpu[1])
-      sampler.emit(:derive, "os.cpu.system", cpu[2])
-      sampler.emit(:derive, "os.cpu.idle", cpu[3])
-      sampler.emit(:derive, "os.cpu.iowait", cpu[4])
-      sampler.emit(:derive, "os.cpu.irq", cpu[5])
-      sampler.emit(:derive, "os.cpu.softirq", cpu[6])
+      Metriks.derive("os.cpu.user").mark(cpu[0])
+      Metriks.derive("os.cpu.nice").mark(cpu[1])
+      Metriks.derive("os.cpu.system").mark(cpu[2])
+      Metriks.derive("os.cpu.idle").mark(cpu[3])
+      Metriks.derive("os.cpu.iowait").mark(cpu[4])
+      Metriks.derive("os.cpu.irq").mark(cpu[5])
+      Metriks.derive("os.cpu.softirq").mark(cpu[6])
     end
   end
 end

data/lib/ganymed/collectors/disk.rb

@@ -16,10 +16,10 @@ collect do
       next if config.exclude.map {|e| Regexp.new(e).match(name)}.any?
 
       block_pc = 1.0 - (st.blocks_free.to_f / st.blocks.to_f)
-      sampler.emit(:gauge, "os.disk.#{name}.blocks", block_pc)
+      Metriks.histogram("os.disk.#{name}.blocks").update(block_pc)
 
       files_pc = 1.0 - (st.files_free.to_f / st.files.to_f)
-      sampler.emit(:gauge, "os.disk.#{name}.files", files_pc)
+      Metriks.histogram("os.disk.#{name}.files").update(files_pc)
     end
   end
 end

data/lib/ganymed/collectors/iostat.rb

@@ -11,8 +11,8 @@ collect do
       ios = Struct::IOStat.new(*line.strip.split(/\s+/))
       next if config.skip_numbered.tap{} and ios.dev =~ /\d+$/
       next if config.exclude.map {|e| Regexp.new(e).match(ios.dev)}.any?
-      sampler.emit(:derive, "os.iostat.#{ios.dev}.rsect", ios.rsect)
-      sampler.emit(:derive, "os.iostat.#{ios.dev}.wsect", ios.wsect)
+      Metriks.derive("os.iostat.#{ios.dev}.rsect").mark(ios.rsect)
+      Metriks.derive("os.iostat.#{ios.dev}.wsect").mark(ios.wsect)
     end
   end
 end

data/lib/ganymed/collectors/load.rb

@@ -2,6 +2,6 @@ collect do
   next if not File.readable?('/proc/loadavg')
   File.open('/proc/loadavg') do |file|
     loadavg = file.read.chomp.split[0,3].map(&:to_f)
-    sampler.emit(:gauge, "os.loadavg", loadavg[0])
+    Metriks.histogram("os.loadavg").update(loadavg[0])
   end
 end

data/lib/ganymed/collectors/memory.rb

@@ -14,28 +14,28 @@ collect do
       when "MemTotal"
         apps = value
       when "MemFree"
-        sampler.emit(:gauge, "os.mem.unused", value)
+        Metriks.histogram("os.mem.unused").update(value)
         apps -= value
       when "Buffers"
-        sampler.emit(:gauge, "os.mem.buffers", value)
+        Metriks.histogram("os.mem.buffers").update(value)
         apps -= value
       when "Cached"
-        sampler.emit(:gauge, "os.mem.cache", value)
+        Metriks.histogram("os.mem.cache").update(value)
         apps -= value
       when "SwapCached"
-        sampler.emit(:gauge, "os.mem.swap_cache", value)
+        Metriks.histogram("os.mem.swap_cache").update(value)
         apps -= value
       when "Slab"
-        sampler.emit(:gauge, "os.mem.slab_cache", value)
+        Metriks.histogram("os.mem.slab_cache").update(value)
         apps -= value
       when "PageTables"
-        sampler.emit(:gauge, "os.mem.page_tables", value)
+        Metriks.histogram("os.mem.page_tables").update(value)
         apps -= value
       when "Mapped"
-        sampler.emit(:gauge, "os.mem.mapped", value)
+        Metriks.histogram("os.mem.mapped").update(value)
       end
     end
 
-    sampler.emit(:gauge, "os.mem.apps", apps)
+    Metriks.histogram("os.mem.apps").update(apps)
   end
 end

data/lib/ganymed/collectors/network.rb

@@ -15,12 +15,12 @@ collect do
 
       next if config.exclude.map {|e| Regexp.new(e).match(ns.dev)}.any?
 
-      sampler.emit(:derive, "os.net.#{ns.dev}.rbytes", ns.rbytes)
-      sampler.emit(:derive, "os.net.#{ns.dev}.wbytes", ns.wbytes)
-      sampler.emit(:derive, "os.net.#{ns.dev}.rpackets", ns.rpackets)
-      sampler.emit(:derive, "os.net.#{ns.dev}.wpackets", ns.wpackets)
-      sampler.emit(:derive, "os.net.#{ns.dev}.rerrs", ns.rerrs)
-      sampler.emit(:derive, "os.net.#{ns.dev}.werrs", ns.werrs)
+      Metriks.derive("os.net.#{ns.dev}.rbytes").mark(ns.rbytes.to_i)
+      Metriks.derive("os.net.#{ns.dev}.wbytes").mark(ns.wbytes.to_i)
+      Metriks.derive("os.net.#{ns.dev}.rpackets").mark(ns.rpackets.to_i)
+      Metriks.derive("os.net.#{ns.dev}.wpackets").mark(ns.wpackets.to_i)
+      Metriks.derive("os.net.#{ns.dev}.rerrs").mark(ns.rerrs.to_i)
+      Metriks.derive("os.net.#{ns.dev}.werrs").mark(ns.werrs.to_i)
     end
   end
 end

data/lib/ganymed/collectors/process.rb

@@ -5,13 +5,13 @@ collect do
       key, value = line.chomp.split
       case key
       when "ctxt"
-        sampler.emit(:derive, "os.procs.switch", value.to_i)
+        Metriks.derive("os.procs.switch").mark(value.to_i)
       when "processes"
-        sampler.emit(:derive, "os.procs.forks", value.to_i)
+        Metriks.derive("os.procs.forks").mark(value.to_i)
       when "procs_running"
-        sampler.emit(:gauge, "os.procs.running", value.to_i)
+        Metriks.histogram("os.procs.running").update(value.to_i)
       when "procs_blocked"
-        sampler.emit(:gauge, "os.procs.blocked", value.to_i)
+        Metriks.histogram("os.procs.blocked").update(value.to_i)
       end
     end
   end

data/lib/ganymed/config.yml

@@ -7,38 +7,6 @@ generic:
     gcprofiler: false
     rubyprof: false
 
-  processor:
-    enabled: true
-    listen:
-      host: 0.0.0.0
-      port: 1336
-    mongodb:
-      host: localhost
-      port: 27017
-      database: ganymed
-      pool_size: 25
-      pool_timeout: 5
-    websocket:
-      host: 0.0.0.0
-      port: 1338
-
-  sampler:
-    enabled: true
-    window: 300
-    listen:
-      host: 127.0.0.1
-      port: 1337
-    consolidations:
-      avg: [mean]
-
-  client:
-    processor:
-      host: 127.0.0.1
-      port: 1336
-    sampler:
-      host: 127.0.0.1
-      port: 1337
-
   collectors:
     network:
       exclude:

data/lib/ganymed/master.rb

@@ -2,14 +2,14 @@ require 'eventmachine'
 require 'madvertise/ext/config'
 require 'mixlib/cli'
 require 'servolux'
-require 'socket'
 
 require 'ganymed'
 require 'ganymed/collector'
-require 'ganymed/processor'
-require 'ganymed/sampler'
 require 'ganymed/version'
 
+require 'metriks'
+require 'metriks/reporter/logger'
+
 module Ganymed
   # @private
   class Master < ::Servolux::Server
@@ -20,9 +20,6 @@ module Ganymed
       @default_config_file = File.join(LIB_DIR, 'ganymed/config.yml')
       @config_file = cli[:config_file]
 
-      # store fqdn for later use
-      config.fqdn = ::Socket.gethostbyname(::Socket.gethostname).first
-
       # initialize servolux
       super('ganymed',
             :interval => 1,
@@ -39,10 +36,14 @@ module Ganymed
 
       EventMachine.epoll # use epoll
       EventMachine.run do
-        Processor.new(config) if config.processor.enabled
-        Sampler.new(config) if config.sampler.enabled
         Collector.new(config) if config.collectors.any?
+        Metriks::Reporter::Logger.new({
+          :logger => log,
+          :interval => 10,
+          :on_error => ->(e) { log.exception(e) },
+        }).start
       end
+
     rescue Exception => exc
       log.exception(exc)
     end

data/lib/ganymed/version.rb

@@ -1,4 +1,4 @@
 module Ganymed
   # @private
-  VERSION = '0.3.4'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ganymed
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-05 00:00:00.000000000 Z
+date: 2012-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -43,22 +43,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.12.10
-- !ruby/object:Gem::Dependency
-  name: ganymed-client
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.2.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: madvertise-ext
   requirement: !ruby/object:Gem::Requirement
@@ -107,22 +91,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: RubyInline
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: servolux
   requirement: !ruby/object:Gem::Requirement
@@ -140,7 +108,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: msgpack
+  name: metriks
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -156,39 +124,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mongo
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '1.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '1.6'
-- !ruby/object:Gem::Dependency
-  name: em-websocket
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: yajl-ruby
+  name: metriks-derive
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -235,55 +171,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.6.12
-- !ruby/object:Gem::Dependency
-  name: ripl
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: ascii_charts
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: terminal-table
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-description: Ganymed is an event collection daemon
+description: Ganymed is a daemon that collects Metriks from third-party applications
 email:
 - bb@xnull.de
 executables:
@@ -302,8 +190,6 @@ files:
 - Rakefile
 - bin/ganymed
 - bin/ganymed-console
-- contrib/cpuhog
-- contrib/cpuhog.c
 - ganymed.gemspec
 - lib/ganymed.rb
 - lib/ganymed/collector.rb
@@ -314,33 +200,9 @@ files:
 - lib/ganymed/collectors/memory.rb
 - lib/ganymed/collectors/network.rb
 - lib/ganymed/collectors/process.rb
-- lib/ganymed/collectors/uptime.rb
 - lib/ganymed/config.yml
-- lib/ganymed/console.rb
-- lib/ganymed/event.rb
-- lib/ganymed/ext/array.rb
 - lib/ganymed/master.rb
-- lib/ganymed/mongodb.rb
-- lib/ganymed/processor.rb
-- lib/ganymed/sampler.rb
-- lib/ganymed/sampler/counter.rb
-- lib/ganymed/sampler/datasource.rb
-- lib/ganymed/sampler/derive.rb
-- lib/ganymed/sampler/gauge.rb
 - lib/ganymed/version.rb
-- lib/ganymed/websocket.rb
-- lib/ganymed/websocket/authentication.rb
-- lib/ganymed/websocket/connection.rb
-- lib/ganymed/websocket/filter.rb
-- lib/ganymed/websocket/metadata.rb
-- lib/ganymed/websocket/query.rb
-- lib/ganymed/websocket/subscribe.rb
-- spec/sampler/counter_spec.rb
-- spec/sampler/datasource_examples.rb
-- spec/sampler/datasource_spec.rb
-- spec/sampler/derive_spec.rb
-- spec/sampler/gauge_spec.rb
-- spec/sampler_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
 - tasks/reek.rake
@@ -360,7 +222,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1351470773654091020
+      hash: 1424796383763485313
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -369,20 +231,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1351470773654091020
+      hash: 1424796383763485313
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: Ganymed is an event collection daemon
+summary: Ganymed is a daemon that collects Metriks from third-party applications
 test_files:
-- spec/sampler/counter_spec.rb
-- spec/sampler/datasource_examples.rb
-- spec/sampler/datasource_spec.rb
-- spec/sampler/derive_spec.rb
-- spec/sampler/gauge_spec.rb
-- spec/sampler_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
 has_rdoc: 

data/contrib/cpuhog.c

@@ -1,21 +0,0 @@
-
-/* do some CPU hogging */
-
-
-int main(int argc, char *argv[])
-{
-  	int i;
-	unsigned long long v1,v2,v3;
-
-	while (1) {
-		v1 = 42;
-		v2 = v3 = -1;
-		for (i=0; i<1000; i++) {
-			v1 += i;
-			v2 = v1*i + v3;
-			v3 = v1 ^ v2;
-		}
-	}
-	return 0;
-}
-

data/lib/ganymed/collectors/uptime.rb

@@ -1,10 +0,0 @@
-collect(900) do
-  next if not File.readable?('/proc/stat')
-  File.open('/proc/stat') do |file|
-    file.each do |line|
-      next if not line =~ /^btime /
-      boottime = Time.at(line.chomp.split[1].to_i)
-      processor.event("os.reboot", 1, :now => boottime)
-    end
-  end
-end