fluent-plugin-librato-metrics 0.2.2 → 0.2.3

data/README.rdoc

@@ -18,34 +18,76 @@ Simply use RubyGems:
       librato_user YOUR_LIBRATO_USER_NAME
       librato_token YOUR_LIBRATO_TOKEN
 
-      # simple key-value metrics
-      <metrics metrics.status.**>
-        name status
-        each_key key
-        value_key value
-      </metrics>
-
-      # single-stream event count
+      # event count
       <metrics metrics.event.mytask>
         name mytask.count
       </metrics>
 
-      # single-stream event sum
+      # event sum
       <metrics metrics.event.mytask>
         name mytask.size
         value_key size
       </metrics>
 
-      # single-stream event sum; value type is float
+      # event sum with float type
       <metrics metrics.event.mytask>
         name mytask.elapsed
         value_key elapsed
         type float
       </metrics>
 
+      # count with source
+      <metrics metrics.web_event.4xx>
+        name web.status_count
+        source 4xx
+      </metrics>
+      <metrics metrics.web_event.5xx>
+        name web.status_count
+        source 5xx
+      </metrics>
+
+      # sum with source
+      <metrics metrics.web_count>
+        name web.status_count
+        value_key 4xx_count
+        source 4xx
+      </metrics>
+      <metrics metrics.web_count>
+        name web.status_count
+        value_key 5xx_count
+        source 5xx
+      </metrics>
+
+      # key-value metrics (multi-stream)
+      <metrics metrics.status.**>
+        name status  # actual name is status.$record[key]
+        each_key key
+        value_key value
+      </metrics>
+
     </match>
 
-[librato_user (required)] User name of librato metrics
+=== Global configuration
+
+[librato_user STRING (required)] User name of librato metrics
+
+[librato_token STRING (required)] Token of librato metrics
+
+[source STRING] Default source name of metrics
+
+=== <metrics> configuration
+
+[name STRING (required)] Name of this metrics
+
+[source STRING] Source name of this metrics
+
+[each_key STRING] Append 'record[each_key]' to this metrics name
+
+[each_keys COMMA,SEPARATED,STRINGS] Append 'record[key].record[key].record[key]...' to this metrics name
+
+[value_key STIRNG] Use 'record[value]' for the value of this metrics. Default is 1.
+
+[count_key STIRNG] Use 'record[value] * record[count_key]' for the value of this metrics. Default is 1.
 
-[librato_token (required)] Token of librato metrics
+[type STIRNG] 'int' or 'float'. Default is int.
 

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.2.3

data/lib/fluent/plugin/out_librato_metrics.rb

@@ -37,6 +37,17 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
     super
   end
 
+  class LibratoMetricsMetrics < Metrics
+    config_param :source, :string, :default => nil
+  end
+
+  # override
+  def new_metrics(conf, pattern)
+    m = LibratoMetricsMetrics.new(pattern)
+    m.configure(conf)
+    m
+  end
+
   def format(tag, time, record)
     out = ''.force_encoding('ASCII-8BIT')
     each_metrics(tag, time, record) {|d|
@@ -47,26 +58,35 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
         key = "#{name}.#{d.keys.join('.')}"
       end
       partitioned_time = time / 60 * 60
-      [key, partitioned_time, d.value*d.count].to_msgpack(out)
+      value = d.value * d.count
+      source = d.metrics.source
+      [key, partitioned_time, value, source].to_msgpack(out)
     }
     out
   end
 
+  AggregationKey = Struct.new(:name, :time, :source)
+
   def write(chunk)
-    counters = {}  #=> {partitioned_time => {name => CounterAggregator}}
-    gauges = {}    #=> {partitioned_time => {name => GaugeAggregator}}
-
-    chunk.msgpack_each {|key,partitioned_time,value|
-      if m = /^counter\.(.*)$/.match(key)
-        name = m[1]
-        ((counters[partitioned_time] ||= {})[name] ||= CounterAggregator.new).add(value)
-      elsif m = /^gauge\.(.*)$/.match(key)
-        name = m[1]
-        ((gauges[partitioned_time] ||= {})[name] ||= GaugeAggregator.new).add(value)
-      else
+    counters = {}  #=> {AggregationKey => XxxAggregator}
+    gauges = {}    #=> {AggregationKey => XxxAggregator}
+
+    chunk.msgpack_each {|key,time,value,source|
+      #if m = /^counter\.(.*)$/.match(key)
+      #  # FIXME what's the actual behavior of counters?
+      #  name = m[1]
+      #  k = AggregationKey.new(name, time, source)
+      #  (counters[k] ||= MaxAggregator.new).add(value)
+      #elsif m = /^gauge\.(.*)$/.match(key)
+      #  # FIXME what's the actual behavior of gauges?
+      #  name = m[1]
+      #  k = AggregationKey.new(name, time, source)
+      #  (gauges[k] ||= AverageAggregator.new).add(value)
+      #else
         name = key
-        ((gauges[partitioned_time] ||= {})[name] ||= GaugeAggregator.new).add(value)
-      end
+        k = AggregationKey.new(name, time, source)
+        (gauges[k] ||= SumAggregator.new).add(value)
+      #end
     }
 
     #http = Net::HTTP.new('metrics-api.librato.com', 80)
@@ -78,18 +98,19 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
     header = {}
 
     begin
-      {'counters'=>counters, 'gauges'=>gauges}.each_pair {|type,partitions|
-        partitions.each_pair {|partitioned_time,name_aggrs|
+      {'counters'=>counters, 'gauges'=>gauges}.each_pair {|type,k_aggrs|
+        # send upto 10 entries at once
+        k_aggrs.each_slice(10) {|k_aggrs_slice|
           req = Net::HTTP::Post.new('/v1/metrics', header)
           req.basic_auth @librato_user, @librato_token
 
-          params = {
-            'measure_time' => partitioned_time.to_s,
-          }
-          params['source'] = @source if @source
+          params = {}
+          params['source'] = @source if @source  # default source
 
-          name_aggrs.each_with_index {|(name,aggr),i|
-            params["#{type}[#{i}][name]"] = name
+          k_aggrs_slice.each_with_index {|(k,aggr),i|
+            params["#{type}[#{i}][name]"] = k.name
+            params["#{type}[#{i}][measure_time]"] = k.time.to_s
+            params["#{type}[#{i}][source]"] = k.source if k.source
             params["#{type}[#{i}][value]"] = aggr.get.to_s
           }
 
@@ -110,7 +131,7 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
 
 
   # max(value) aggregator
-  class CounterAggregator
+  class MaxAggregator
     def initialize
       @value = 0
     end
@@ -126,8 +147,23 @@ class LibratoMetricsOutput < Fluent::BufferedOutput
     end
   end
 
+  # sum(value) aggregator
+  class SumAggregator
+    def initialize
+      @value = 0
+    end
+
+    def add(value)
+      @value += value
+    end
+
+    def get
+      @value
+    end
+  end
+
   # avg(value) aggregator
-  class GaugeAggregator
+  class AverageAggregator
     def initialize
       @value = 0
       @num = 0

data/lib/fluent/plugin/out_librato_metrics_mixin.rb

@@ -46,7 +46,7 @@ module LibratoMetricsOutput::MetricsMixin
       e.name == 'metrics'
     }.each {|e|
       defaults.each_pair {|k,v| e[k] = v if v != nil }
-      add_metrics(e, e.arg)
+      @metrics << new_metrics(e, e.arg)
     }
 
     if mk = @match_key
@@ -56,10 +56,9 @@ module LibratoMetricsOutput::MetricsMixin
     end
   end
 
-  def add_metrics(conf, pattern)
+  def new_metrics(conf, pattern)
     m = Metrics.new(pattern)
     m.configure(conf)
-    @metrics << m
   end
 
   class Data
@@ -185,18 +184,25 @@ module LibratoMetricsOutput::MetricsMixin
     def evaluate(tag, time, record)
       return nil unless @match_proc.call(tag)
 
-      data = @data
-
-      keys = data.keys = @key_proc.call(record)
+      keys = @key_proc.call(record)
       return nil unless keys
 
-      value = data.value = @value_proc.call(record)
+      value = @value_proc.call(record)
       return nil unless value
 
-      data.count = @count_proc.call(record)
+      count = @count_proc.call(record)
+
+      data = new_data
+      data.keys = keys
+      data.value = value
+      data.count = count
 
       return data
     end
+
+    def new_data
+      Data.new(self)
+    end
   end
 
   def each_metrics(tag, time, record, &block)

data/test/out_librato_metrics.rb

@@ -36,6 +36,13 @@ class LibratoMetricsOutputTest < Test::Unit::TestCase
       value_key elapsed
       type float
     </metrics>
+
+    # single-stream event sum with source
+    <metrics test5.**>
+      name test5.stack
+      value_key count
+      source type1
+    </metrics>
   ]
 
   TIME = Time.parse("2011-01-02 13:14:15 UTC").to_i
@@ -57,23 +64,28 @@ class LibratoMetricsOutputTest < Test::Unit::TestCase
   def test_format
     d = create_driver("test1.1")
     d.emit({"key"=>"a", "value"=>10, "count"=>2}, TIME)
-    d.expect_format ["test1.a", TIME/60*60, 20].to_msgpack
+    d.expect_format ["test1.a", TIME/60*60, 20, nil].to_msgpack
     d.run
 
     d = create_driver("test2.2")
     d.emit({}, TIME)
-    d.expect_format ["test2.count", TIME/60*60, 1].to_msgpack
+    d.expect_format ["test2.count", TIME/60*60, 1, nil].to_msgpack
     d.run
 
     d = create_driver("test3.3")
     d.emit({"size"=>2100}, TIME)
-    d.expect_format ["test3.size", TIME/60*60, 2100].to_msgpack
+    d.expect_format ["test3.size", TIME/60*60, 2100, nil].to_msgpack
     d.run
 
     d = create_driver("test3.test4")
     d.emit({"size"=>2400, "elapsed"=>10.0}, TIME)
-    d.expect_format ["test3.size", TIME/60*60, 2400].to_msgpack
-    d.expect_format ["test4.elapsed", TIME/60*60, 10.0].to_msgpack
+    d.expect_format ["test3.size", TIME/60*60, 2400, nil].to_msgpack
+    d.expect_format ["test4.elapsed", TIME/60*60, 10.0, nil].to_msgpack
+    d.run
+
+    d = create_driver("test5")
+    d.emit({"count"=>12}, TIME)
+    d.expect_format ["test5.stack", TIME/60*60, 12, "type1"].to_msgpack
     d.run
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-librato-metrics
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-30 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &70131020756540 !ruby/object:Gem::Requirement
+  requirement: &70308779156220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.10.0
   type: :runtime
   prerelease: false
-  version_requirements: *70131020756540
+  version_requirements: *70308779156220
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70131020756060 !ruby/object:Gem::Requirement
+  requirement: &70308779155740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70131020756060
+  version_requirements: *70308779155740
 description: Librato metrics output plugin for Fluent event collector
 email:
 - frsyuki@gmail.com
@@ -64,7 +64,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -197996435032986197
+      hash: 3018102368853295399
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -73,7 +73,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -197996435032986197
+      hash: 3018102368853295399
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.12