ganymed 0.3.4 → 0.4.0

data/lib/ganymed/console.rb

@@ -1,147 +0,0 @@
-require 'ascii_charts'
-require 'digest'
-require 'madvertise/ext/config'
-require 'ripl'
-require 'terminal-table'
-
-require 'ganymed'
-require 'ganymed/mongodb'
-
-module Ganymed
-  class Console
-    include Configuration::Helpers
-
-    attr_accessor :db
-
-    def initialize
-      @cache = {}
-
-      # load config file
-      @default_config_file = File.join(LIB_DIR, 'ganymed/config.yml')
-
-      # store fqdn for later use
-      config.fqdn = ::Socket.gethostbyname(::Socket.gethostname).first
-
-      # initialize db before loading ripl
-      log.silence do
-        @db = Ganymed::MongoDB.new(config.processor.mongodb)
-      end
-
-      Ripl.start(binding: binding)
-    end
-
-    def adduser(username, password)
-      hashed = Digest::SHA256.hexdigest(password)
-      db['users'].find_and_modify(
-        query: {_id: username},
-        update: {password: hashed},
-        new: true
-      )
-    end
-
-    def origins
-      headings = ["Origin"]
-      rows = db['metadata'].find.to_a.map do |row|
-        [row['_id']]
-      end
-      puts Terminal::Table.new(rows: rows, headings: headings)
-    end
-
-    alias_method :o, :origins
-
-    def list
-      db.collection_names.reject do |col|
-        %w(
-          metadata
-          system.indexes
-          users
-        ).include?(col)
-      end.sort.each do |col|
-        puts col
-      end
-      nil
-    end
-
-    alias_method :ls, :list
-    alias_method :l, :list
-
-    def drop
-      db.connection.drop_database(config.processor.mongodb.database)
-    end
-
-    def method_missing(method, *args)
-      pattern = /^#{method}\./
-
-      namespaces = db.collection_names.select do |col|
-        col =~ pattern
-      end
-
-      if namespaces.empty?
-        super
-      else
-        @cache[method] ||= CollectionProxy.new(method, db, config)
-      end
-    end
-
-    class CollectionProxy
-      attr_reader :ns, :db, :config
-
-      def initialize(ns, db, config)
-        @ns, @db, @config = ns, db, config
-        @cache = {}
-        @consolidation = "avg"
-
-        metaclass = class << self; self; end
-        config.sampler.consolidations.each do |key, _|
-          metaclass.send(:define_method, key.to_sym) do
-            @consolidation = key
-            self
-          end
-        end
-      end
-
-      def top(limit=10, selector={})
-        selector.merge!({c: { :$in => [nil, @consolidation]}})
-        opts = {limit: limit, sort: [:t, -1]}
-        db[ns].find(selector, opts).to_a.map do |row|
-          row.delete('_id')
-          row.delete('c')
-          row['t'] = Time.at(row['t'])
-          row
-        end
-      end
-
-      def to_s
-        headings = ["Time", "Origin", "Value"]
-        rows = top.map do |row|
-          [row['t'], row['o'], row['v']]
-        end
-        table = Terminal::Table.new(rows: rows, headings: headings)
-        "top-10 most recent events:\n" + table.to_s
-      end
-
-      def chart(limit=10, origin=nil)
-        origin ||= config.fqdn
-        values = top(limit, {o: origin}).reverse.map do |event|
-          [event['t'].strftime("%H:%M"), event['v']]
-        end
-        puts AsciiCharts::Cartesian.new(values, bar: true).draw
-      end
-
-      def method_missing(method, *args)
-        subns = "#{@ns}.#{method}"
-        pattern = /^#{subns}(\.|$)/
-
-        namespaces = db.collection_names.select do |col|
-          col =~ pattern
-        end
-
-        if namespaces.empty?
-          db[ns].send(method, *args)
-        else
-          @cache[method] ||= CollectionProxy.new(subns, db, config)
-        end
-      end
-    end
-  end
-end

data/lib/ganymed/event.rb

@@ -1,97 +0,0 @@
-module Ganymed
-
-  ##
-  # An incoming event is a hash-like object and contains the following keys:
-  #
-  # n::  namespace (e.g. +os.cpu.idle+, +nginx.requests+, etc)
-  # c::  a consolidation function used to build this event from multiple
-  #      samples (_optional_)
-  # o::  origin (e.g. +web1.example.com+, +myapplication+)
-  # t::  timestamp as seconds since epoch
-  # r::  resolution of this event (i.e. how often is this event sent to the processor, _optional_)
-  # v::  the event value (any object that can be serialized with BSON and JSON)
-  #
-  class Event
-    # The namespace of this event.
-    # @return [String]
-    attr_accessor :ns
-
-    # The consolidation function used to build this event from multiple
-    # samples if any.
-    # @return [String]
-    attr_accessor :cf
-
-    # The origin of this event.
-    # @return [String]
-    attr_accessor :origin
-
-    # The timestamp of this event.
-    # @return [Fixnum]
-    attr_accessor :timestamp
-
-    # The event resolution
-    # @return [Fixnum]
-    #
-    # @note +nil+ or +0+ will bypass range checks for MongoDB, refer to the
-    #       {file:doc/configuration.md configuration} for details.
-    attr_accessor :resolution
-
-    # The actual value of this sample.
-    # @return [Object]
-    attr_accessor :value
-
-    # Create a new {Event} instance from +obj+.
-    #
-    # @return [Event] The parsed event instance.
-    def self.parse(obj)
-      event = new
-      event.ns = obj['n']
-      event.cf = obj['c']
-      event.origin = obj['o']
-      event.timestamp = obj['t'].to_i
-      event.resolution = obj['r'].to_i
-
-      value = obj['v']
-      if value.is_a?(Float) and (value.nan? or value.infinite?)
-        event.value = nil
-      else
-        event.value = value
-      end
-
-      event.validate
-      event
-    end
-
-    # Convert this event into a hash with long keys. This hash cannot be parsed
-    # by #parse which accepts short keys for effiency.
-    #
-    # @return [Hash]
-    def to_h
-      {
-        :ns => ns,
-        :cf => cf,
-        :origin => origin,
-        :timestamp => timestamp,
-        :resolution => resolution,
-        :value => value
-      }
-    end
-
-    # Validate this event.
-    #
-    # @return [void]
-    def validate
-      unless @ns =~ /^[a-z][a-z0-9_\.-]+[a-z0-9]$/i
-        raise "invalid namespace: #{@ns.inspect}"
-      end
-
-      if @ns =~ /^(system\.|metadata)/
-        raise "namespace is reserved: #{@ns.inspect}"
-      end
-
-      if @timestamp < 0
-        raise "invalid timestamp: #{@timestamp.inspect}"
-      end
-    end
-  end
-end

data/lib/ganymed/ext/array.rb

@@ -1,38 +0,0 @@
-require 'inline'
-
-class Array
-  # this is incredibly faster than doing ary.inject(0, :+)
-  # should only be called on numbers-only arrays, though ;-)
-  inline :C do |builder|
-    builder.c_raw <<-EOC
-    static VALUE sum(int argc, VALUE *argv, VALUE self) {
-      double result = 0;
-      VALUE *arr = RARRAY_PTR(self);
-      long i, len = RARRAY_LEN(self);
-
-      for (i = 0; i < len; i++) {
-        result += NUM2DBL(arr[i]);
-      }
-
-      return rb_float_new(result);
-    }
-    EOC
-  end
-
-  def mean
-    sum.to_f / length
-  end
-
-  def variance
-    m = mean
-    reduce(0) {|accum, item| accum + (item - m) ** 2}.to_f / (length - 1)
-  end
-
-  def stdev
-    Math.sqrt(variance)
-  end
-
-  def percentile(pc)
-    sort[(pc * length).ceil - 1]
-  end
-end

data/lib/ganymed/mongodb.rb

@@ -1,44 +0,0 @@
-require 'mongo'
-
-module Ganymed
-  # @private
-  class MongoDB
-    attr_reader :config
-
-    def initialize(config)
-      @config = config
-      @collections = {}
-      log.info("using MongoDB at #{config.host}:#{config.port}/#{config.database}")
-    end
-
-    def connection
-      @connection ||= ::Mongo::Connection.new(config.host, config.port,
-                                              :pool_size => config.pool_size,
-                                              :pool_timeout => config.pool_timeout)
-    end
-
-    def db
-      @db ||= connection.db(config.database)
-    end
-
-    def collection(ns)
-      return @collections[ns] if @collections.has_key?(ns)
-      col = @collections[ns] = db.collection(ns)
-      col.ensure_index([['c', ::Mongo::ASCENDING]])
-      col.ensure_index([['o', ::Mongo::ASCENDING]])
-      col.ensure_index([['t', ::Mongo::ASCENDING]])
-      col.ensure_index([['c', ::Mongo::ASCENDING], ['o', ::Mongo::ASCENDING]])
-      col.ensure_index([['o', ::Mongo::ASCENDING], ['t', ::Mongo::ASCENDING]])
-      col.ensure_index([['c', ::Mongo::ASCENDING], ['o', ::Mongo::ASCENDING], ['t', ::Mongo::ASCENDING]], :unique => true)
-      col
-    end
-
-    def [](*args)
-      db.collection(*args)
-    end
-
-    def method_missing(method, *args)
-      db.send(method, *args)
-    end
-  end
-end

data/lib/ganymed/processor.rb

@@ -1,137 +0,0 @@
-require 'eventmachine'
-require 'msgpack'
-
-require 'ganymed/client'
-require 'ganymed/event'
-require 'ganymed/mongodb'
-require 'ganymed/websocket'
-
-module Ganymed
-
-  ##
-  # The Processor accepts {Event events} from a UDP socket and stores these
-  # metrics into MongoDB. All metrics are also published to interested parties
-  # via WebSocket.
-  #
-  class Processor
-    def initialize(config)
-      @config = config
-      @db = MongoDB.new(config.processor.mongodb)
-      @websocket = Websocket.new(config.processor.websocket, @db)
-
-      @config.client.sampler.tap do |sampler|
-        # schedule connect on next tick so that our sampler socket is
-        # available. Processor#initialize is called from EM.run and therefore
-        # no sockets have been opened yet.
-        EM.next_tick { @sampler = Ganymed::Client::Sampler.connect(sampler.host, sampler.port) }
-      end
-
-      cache!
-      listen!
-    end
-
-    def cache!
-      @metadata = Hash.new do |hsh, key|
-        hsh[key] = []
-      end
-
-      @db['metadata'].find({}, fields: [:_id, :namespaces]).each do |result|
-        @metadata[result['_id']] = result['namespaces']
-      end
-    end
-
-    def listen!
-      @config.processor.listen.tap do |listen|
-        log.info("processing metrics on udp##{listen.host}:#{listen.port}")
-        EM.open_datagram_socket(listen.host, listen.port, Connection) do |connection|
-          connection.processor = self
-        end
-
-        log.info("processing metrics on tcp##{listen.host}:#{listen.port}")
-        EM.start_server(listen.host, listen.port, Connection) do |connection|
-          connection.processor = self
-        end
-      end
-    end
-
-    def process(data)
-      case data['_type'].to_sym
-      when :event
-        process_event(Event.parse(data))
-      when :metadata
-        process_metadata(data)
-      end
-    rescue Exception => exc
-      log.exception(exc)
-    end
-
-    def process_event(event)
-      @sampler.emit(:counter, 'ganymed.processor.events', 1) rescue nil
-
-      if not @metadata[event.origin].include?(event.ns)
-        # insert origin + namespace into metadata
-        metadata = @db['metadata'].find_and_modify({
-          query: {:_id => event.origin},
-          update: {:$addToSet => {:namespaces => event.ns}},
-          upsert: true,
-        })
-
-        # set defaults in case find_and_modify returned an empty document
-        metadata['_id'] ||= event.origin
-        metadata['namespaces'] ||= [event.ns]
-
-        # notify websocket clients
-        @websocket.send(:metadata, [metadata])
-      end
-
-      # normalize timestamp for events with a resolution
-      if event.resolution > 1
-        event.timestamp -= event.timestamp % event.resolution
-      end
-
-      # insert event into ns collection
-      @db.collection(event.ns).update({
-        :c => event.cf,
-        :o => event.origin,
-        :t => event.timestamp
-      }, {
-        :$set => {:v => event.value}
-      }, :upsert => true)
-
-      # send the event to websocket clients
-      @websocket.each do |connection|
-        connection.publish(event)
-      end
-    end
-
-    def process_metadata(data)
-      metadata = @db['metadata'].find_and_modify({
-        query: {:_id => data['origin']},
-        update: {:$set => {:data => data['data']}},
-        upsert: true,
-        new: true
-      })
-
-      @websocket.send(:metadata, [metadata])
-    end
-
-    # @private
-    class Connection < EM::Connection
-      attr_accessor :processor
-
-      def initialize
-        @pac = MessagePack::Unpacker.new
-      end
-
-      def receive_data(data)
-        @pac.feed(data)
-        @pac.each do |obj|
-          EM.defer { processor.process(obj) }
-        end
-      rescue Exception => exc
-        log.exception(exc)
-        close_connection
-      end
-    end
-  end
-end

data/lib/ganymed/sampler/counter.rb

@@ -1,19 +0,0 @@
-require 'ganymed/ext/array'
-require 'ganymed/sampler/datasource'
-
-module Ganymed
-  class Sampler
-
-    ##
-    # A Counter {DataSource} processes samples counting a specific event (such
-    # as a webserver request, user login, etc) and produces a rate/s gauge.
-    #
-    class Counter < DataSource
-      def flush
-        each do |ns, origin, ts|
-          yield ns, origin, ts.values.map(&:sum)
-        end
-      end
-    end
-  end
-end

data/lib/ganymed/sampler/datasource.rb

@@ -1,85 +0,0 @@
-require 'inline'
-
-module Ganymed
-  class Sampler
-
-    ##
-    # A DataSource processes samples from various sources.  Samples are stored
-    # to an in-memory buffer using a sliding-window algorithm. Values are kept
-    # for a configurable time and consolidated values can be flushed to the
-    # processor at any time.
-    #
-    # All DataSources interpolate their samples to gauge style values before
-    # flushing to the processor, e.g. a counter (like requests, or bytes sent
-    # over the network) is interpolated to a rate/s gauge (e.g. requests/s or
-    # bytes/s).
-    #
-    class DataSource
-
-      attr_reader :buffer, :window
-
-      # Create a new DataSource.
-      #
-      # @param [Fixnum] window  Sliding window size.
-      def initialize(window)
-        @window = window
-        @buffer = Hash.new do |buffer, origin|
-          buffer[origin] = Hash.new do |origin, ns|
-            origin[ns] = Hash.new do |ns, timestamp|
-              ns[timestamp] = []
-            end
-          end
-        end
-      end
-
-      # Feed data from the source.
-      #
-      # @param [String] ns  The namespace for this value.
-      # @param [String] origin The origin of this value.
-      # @param [Fixnum,Float] value  The actual value.
-      # @return [void]
-      def feed(ns, origin, value)
-        @buffer[origin][ns][now] << value
-      end
-
-      # Flush and consolidate the buffer.
-      #
-      # @yield [ns, origin, values]  Run the block once for every
-      #                              namespace/origin, passing in all the
-      #                              consolidated values.
-      # @return [void]
-      def flush
-        raise NotImplementedError
-      end
-
-      # Yield values for every origin/ns combination and remove old elements
-      # from the buffer (sliding window). This is used internally to iterate
-      # through the values in the current window.
-      #
-      # @yield [ns, origin, ts]  Run the block once for every namespace
-      #                          passing in all the collected samples grouped
-      #                          by timestamp.
-      def each(&block)
-        oldest = now - window
-        @buffer.each do |origin, ns|
-          ns.each do |ns, ts|
-            ts.delete_if { |ts, _| ts < oldest }
-            yield ns, origin, ts
-          end
-        end
-      end
-
-      # Time.now.utc.to_i is way too expensive
-      inline :C do |builder|
-        builder.include "<sys/time.h>"
-        builder.c <<-EOC
-        long now(void) {
-          struct timeval tv;
-          gettimeofday(&tv, NULL);
-          return tv.tv_sec;
-        }
-        EOC
-      end
-    end
-  end
-end

data/lib/ganymed/sampler/derive.rb

@@ -1,27 +0,0 @@
-require 'ganymed/sampler/datasource'
-
-module Ganymed
-  class Sampler
-
-    ##
-    # A Derive {DataSource data source} will store the derivative of the line
-    # going from the last to the current value of the data source. Derive is
-    # similar to a {Counter} {DataSource data source} in that it produces a
-    # rate/s gauge, but only accepts absolute values and produces the
-    # derivative instead of summation of all samples.
-    #
-    class Derive < DataSource
-      def flush
-        each do |ns, origin, ts|
-          values = ts.each_cons(2).map do |a, b|
-            (b[1].last - a[1].last)/(b[0] - a[0]) # ∆value / ∆ts
-          end.reject do |value|
-            value < 0 # ignore overflow/counter wrap
-          end
-
-          yield ns, origin, values
-        end
-      end
-    end
-  end
-end

data/lib/ganymed/sampler/gauge.rb

@@ -1,20 +0,0 @@
-require 'ganymed/sampler/datasource'
-
-module Ganymed
-  class Sampler
-
-    ##
-    # A Gauge is the simplest DataSource type. It simply records the given
-    # value in the buffer and emits all values in the buffer upon flush
-    # assuming the given values are in gauge-style (e.g. free memory, users
-    # currently logged in, etc).
-    #
-    class Gauge < DataSource
-      def flush
-        each do |ns, origin, ts|
-          yield ns, origin, ts.values.flatten
-        end
-      end
-    end
-  end
-end