monitoring_protocols 0.0.4

data/lib/monitoring_protocols/core.rb

@@ -0,0 +1,23 @@
+module MonitoringProtocols
+  @@parsers  = {}
+  @@builders = {}
+  
+  def self.register_parser(name, parser_class)
+    @@parsers[name] = parser_class
+  end
+  
+  def self.register_builder(name, builder_class)
+    @@builders[name] = builder_class
+  end
+  
+  def self.get_parser(protocol)
+    klass = @@parsers[protocol.to_sym]
+    klass ? klass.new : nil
+  end
+  
+  def self.get_builder(protocol)
+    klass = @@builders[protocol.to_sym]
+    klass ? klass.new : nil
+  end
+  
+end

data/lib/monitoring_protocols/data_struct.rb

@@ -0,0 +1,98 @@
+
+require 'bigdecimal'
+
+class BigDecimal
+  def to_msgpack(pack)
+    pack.write(to_f)
+  end
+end
+
+module MonitoringProtocols
+  class DataStruct
+    
+    include Comparable
+    
+    def initialize(*args)
+      merge_data_from!(*args)
+    end
+    
+    ##
+    # Merge new data in the structure.
+    #
+    # @param [Object,Hash] opts_or_obj Source
+    # @param [Array] only_fields an array of symbol
+    #   specifying which fields to copy
+    # @param [Boolean] allow_nil If false nil values from
+    #   the source will not be copied in object
+    #
+    def merge_data_from!(opts_or_obj = {}, only_fields = nil, allow_nil = false)
+      keys_left = list_keys(opts_or_obj)
+      
+      self.class.attributes.select{|attr_name| selected_field?(attr_name, only_fields) }.each do |attr_name|
+        v = opts_or_obj.is_a?(Hash) ? (opts_or_obj[attr_name.to_s] || opts_or_obj[attr_name]) : opts_or_obj.send(attr_name)
+        if allow_nil || !v.nil?
+          send("#{attr_name}=", v)
+        end
+        
+        keys_left.delete(attr_name)
+      end
+      
+      unless keys_left.empty?
+        raise ArgumentError, "unknown keys: #{keys_left}"
+      end
+    end
+    
+    def list_keys(what)
+      if what.respond_to?(:keys)
+        what.keys.clone
+      else
+        []
+      end
+    end
+    
+    def selected_field?(field, list)
+      list.nil? || list.include?(field.to_sym)
+    end
+    
+    def to_h
+      h = {}
+      self.class.attributes.each do |attr_name|
+        h[attr_name] = send(attr_name)
+      end
+      h
+    end
+    
+    def to_a
+      self.class.attributes.map{|attr_name| send(attr_name) }
+    end
+    
+    def to_msgpack(pack)
+      pack.write(to_h)
+    end
+    
+    def <=>(other)
+      self.to_a <=> other.to_a
+    end
+
+
+    class <<self
+      def properties(*names)
+        names.each do |name|
+          attr_accessor(name)
+          (@attributes ||= []) << name
+        end
+      end
+      
+      alias :property :properties
+
+      def attributes
+        if (superclass <= DataStruct) && (superclass.attributes)
+          superclass.attributes + @attributes
+        else
+          @attributes
+        end
+      end
+    end
+
+  end
+end

data/lib/monitoring_protocols/json/builder.rb

@@ -0,0 +1,41 @@
+require 'oj'
+
+module MonitoringProtocols
+  module JSON
+    KEYS = %i(host app_name res_name metric_name value).freeze
+    COMPRESSABLE_KEYS = %i(host app_name).freeze
+    
+    class Builder < Builder
+      def build_packet
+        # first we need to find common properties
+        json = {type: 'datapoints'}
+        
+        COMPRESSABLE_KEYS.each do |attr_name|
+          v = @points[0].send(attr_name)
+          if @points.all?{|p| p.send(attr_name) == v }
+            json[attr_name] = v
+          end
+        end
+        
+        points_left = @points
+        
+        
+        # find the root key
+        keys_left = KEYS.select{|key| !json.has_key?(key) }
+        
+        raise "unsupported" unless keys_left.size == 3
+        
+        # now fill the rest
+        until points_left.empty?
+          p = points_left.pop()
+          
+          json[p.res_name] ||= {}
+          json[p.res_name][p.metric_name] = p.value
+        end
+        
+        Oj.dump(json, symbol_keys: false, mode: :compat)
+      end
+    end
+    
+  end
+end

data/lib/monitoring_protocols/json/parser.rb

@@ -0,0 +1,115 @@
+require 'oj'
+
+module MonitoringProtocols
+  module JSON
+  
+    # {
+    #   type: 'datapoints',
+    #   host: 'linux1'
+    #   app_name: 'system',
+    #
+    #   cpu: {
+    #     'user'  => 43,
+    #     'sys'   => 3.4,
+    #     'nice'  => 6.0
+    #   },
+    #
+    #   memory: {
+    #     'total'  => 2048,
+    #     'used'   => 400
+    #   }
+    # }
+    
+    # {
+    #   type: 'datapoints',
+    #   app_name: 'system',
+    #
+    #   linux1: =>
+    #     cpu: {
+    #       'user'  => 43,
+    #       'sys'   => 3.4,
+    #       'nice'  => 6.0
+    #     },
+    #
+    #     memory: {
+    #       'total'  => 2048,
+    #       'used'   => 400
+    #     }
+    #   }
+    # }
+    #
+    class Parser < Parser
+      
+      def self._parse(buffer)
+        Oj.load(buffer, symbol_keys: false)
+      end
+      
+      def self.parse(buffer)
+        packets = []
+        
+        data = _parse(buffer)
+        
+        msg_type = data.delete('type')
+        
+        if msg_type == 'datapoints'
+          parse_datapoints(data) do |pkt|
+            packets << pkt
+          end
+        end
+        
+        packets
+        
+      rescue Oj::ParseError, MonitoringProtocols::ParseError
+        puts "Unable to parse: #{buffer}"
+        []
+      end
+      
+    private
+      def self.recursive_parse(point_data, next_fields, field_index, json_document, &block)
+        root_field = next_fields[field_index]
+        
+        json_document.each do |name, value|
+          if root_field == :metric_name
+            point_data[root_field.to_sym] = name
+            point_data[:value] = parse_and_validate_value(value)
+            
+            msg = DataPoint.new(point_data)
+            block.call(msg)
+          else
+            point_data[root_field] = name
+            recursive_parse(point_data, next_fields, field_index + 1, value, &block)
+          end
+        end
+      end
+      
+      def self.parse_datapoints(data, &block)
+        common_data = {}
+        
+        common_data[:host]     = data.delete('host')
+        common_data[:app_name] = data.delete('app_name')
+        common_data[:res_name] = data.delete('app_name')
+        common_data[:first] = data.delete('first')
+        
+        if time = data.delete('time')
+          common_data[:time] = Time.parse(time)
+        else
+          common_data[:time] = Time.now.utc()
+        end
+        
+        # find which field we expect as the toplevel of the
+        # json document
+        next_fields = [:host, :app_name, :res_name, :metric_name]
+        field_index = 0
+        while (field_index < next_fields.size) && (common_data[next_fields[field_index]] != nil)
+          field_index+= 1
+        end
+        
+        recursive_parse(common_data, next_fields, field_index, data, &block)
+      end
+      
+    end
+    
+  end
+  
+  register_parser(:json, JSON::Parser)
+end

data/lib/monitoring_protocols/msgpack/parser.rb

@@ -0,0 +1,16 @@
+require 'msgpack'
+
+module MonitoringProtocols
+  module MsgPack
+  
+    class Parser < JSON::Parser
+      def self._parse(buffer)
+        MessagePack.unpack(buffer)
+      end
+      
+    end
+    
+  end
+  
+  register_parser(:msgpack, MsgPack::Parser)
+end

data/lib/monitoring_protocols/parser.rb

@@ -0,0 +1,22 @@
+module MonitoringProtocols
+  
+  ParseError = Class.new(RuntimeError)
+  
+  class Parser
+    
+    def parse(data)
+      self.class.parse(data)
+    end
+  
+  private
+    def self.parse_and_validate_value(v)
+      case v
+      when Fixnum, Bignum     then v
+      when Float, BigDecimal  then ((v * 1000).to_i) / 1000.0
+      else
+        raise ParseError, "invalid value: #{v} [#{v.class}]"
+      end
+    end
+    
+  end
+end

data/lib/monitoring_protocols/struct.rb

@@ -0,0 +1,97 @@
+require 'time'
+require 'msgpack'
+require File.expand_path('../data_struct', __FILE__)
+
+module MonitoringProtocols
+  
+  class NetworkMessage < DataStruct
+    
+    ##
+    # Return a generic structure representing
+    # the content of this message.
+    #
+    # @return [DataPoint,Notification]
+    #
+    def convert_content
+      raise 'reimplement in subclass'
+    end
+  end
+  
+  class CommonData < DataStruct
+    properties(
+        :time,
+        :host,
+        :app_name,
+        :res_name,
+        :metric_name,
+      )
+    
+    def time=(val)
+      case val
+      when Time     then @time = val
+      when Numeric  then @time = Time.at(val).getutc()
+      else
+        raise "invalid type for time: #{val}"
+      end
+    end
+    
+    def measure_id(sep = '-')
+      [host, app_name, res_name, metric_name].join(sep)
+    end
+    
+    
+    def to_h
+      super.merge(
+          time: time ? time.iso8601 : nil
+        )
+    end
+    
+    def convert_content
+      [self]
+    end
+
+  end
+  
+  class DataPoint < CommonData
+    properties(
+        :value,
+        :first
+      )
+    
+    def data?; true; end
+    
+    def to_h
+      super.merge(
+          type: 'datapoint'
+        )
+    end
+
+  end
+  
+  class Notification < CommonData
+    SEVERITY = [:info, :warn, :error].freeze
+    
+    properties(
+        :severity,
+        :message
+      )
+    
+    def severity=(val)
+      if val.is_a?(Fixnum)
+        @severity = SEVERITY[val]
+      else
+        @severity = val
+      end
+    end
+    
+    def data?; false; end
+    
+    def to_h
+      super.merge(
+          type: 'notification'
+        )
+    end
+
+  end
+  
+end

data/lib/monitoring_protocols/version.rb

@@ -0,0 +1,3 @@
+module MonitoringProtocols
+  VERSION = "0.0.4"
+end

data/monitoring_protocols.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/monitoring_protocols/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Julien Ammous"]
+  gem.email         = ["schmurfy@gmail.com"]
+  gem.description   = %q{...}
+  gem.summary       = %q{....}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.name          = "monitoring_protocols"
+  gem.require_paths = ["lib"]
+  gem.version       = MonitoringProtocols::VERSION
+  
+  gem.add_dependency 'msgpack'
+  gem.add_dependency 'oj'
+end