fluent-plugin-metricsense 0.1.1 → 0.2.0

data/Gemfile

@@ -1,2 +1,3 @@
-source :rubygems
+source 'https://rubygems.org'
+
 gemspec

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.2.0

data/lib/fluent/plugin/{backend → backends}/librato_backend.rb

@@ -22,7 +22,7 @@ module Fluent::MetricSenseOutput::Backends
   require 'json'
 
   class LibratoBackend < Fluent::MetricSenseOutput::Backend
-    Fluent::MetricSenseOutput::BACKENDS['librato'] = self
+    Fluent::MetricSenseOutput.register_backend('librato', self)
 
     config_param :librato_user, :string
     config_param :librato_token, :string
@@ -48,7 +48,7 @@ module Fluent::MetricSenseOutput::Backends
           req.basic_auth @librato_user, @librato_token
 
           data = []
-          slice.each_with_index {|(tag,time,value,seg_key,seg_val),i|
+          slice.each_with_index {|(tag,time,value,seg_key,seg_val,mode),i|
             if seg_key
               name = "#{tag}:#{seg_key}"
               source = seg_val
@@ -63,7 +63,7 @@ module Fluent::MetricSenseOutput::Backends
             }
             h["source"] = source.to_s if source
             data << h
-            ensure_metric_initialized(http, name)
+            ensure_metric_initialized(http, name, mode)
           }
           body = {"gauges"=>data}.to_json
 
@@ -83,22 +83,32 @@ module Fluent::MetricSenseOutput::Backends
       end
     end
 
-    METRIC_INITIALIZE_REQUEST = [
-      "type=gauge",
-      "attributes[source_aggregate]=true",
-      "attributes[summarize_function]=sum",
-      "attributes[display_stacked]=true",
-    ].join('&')
+    METRIC_INITIALIZE_REQUEST_PER_MODE = []
 
-    def ensure_metric_initialized(http, name)
+    METRIC_INITIALIZE_REQUEST_PER_MODE[UpdateMode::ADD] = {
+        "type" => "gauge",
+        "attributes" => {
+          "source_aggregate" => true,
+          "summarize_function" => "sum",
+        }
+      }.to_json
+
+    METRIC_INITIALIZE_REQUEST_PER_MODE[UpdateMode::LATEST] = {
+        "type" => "gauge",
+        "attributes" => {
+        }
+      }.to_json
+
+    def ensure_metric_initialized(http, name, mode)
       return if @initialized_metrics[name]
 
       header = {}
       req = Net::HTTP::Put.new("/v1/metrics/#{CGI.escape name}", header)
       req.basic_auth @librato_user, @librato_token
 
-      $log.trace { "librato initialize metric: #{name}" }
-      req.body = METRIC_INITIALIZE_REQUEST
+      $log.trace { "librato initialize metric with mode #{mode}: #{name}" }
+      req.body = METRIC_INITIALIZE_REQUEST_PER_MODE[mode]
+      req.set_content_type("application/json")
       res = http.request(req)
 
       # TODO error handling

data/lib/fluent/plugin/{backend → backends}/rdb_tsdb_backend.rb

@@ -18,9 +18,7 @@
 module Fluent::MetricSenseOutput::Backends
 
   class RDBTSDBBackend < Fluent::MetricSenseOutput::Backend
-    include Fluent::Configurable
-
-    Fluent::MetricSenseOutput::BACKENDS['rdb_tsdb'] = self
+    Fluent::MetricSenseOutput.register_backend('rdb_tsdb', self)
 
     config_param :rdb_url, :string
     config_param :rdb_table_prefix, :string
@@ -180,7 +178,8 @@ module Fluent::MetricSenseOutput::Backends
       ensure_connect do |db|
         # group by row_key (base_time,metric_id,segment_id)
         rows = {}
-        data.each {|(tag,time,value,seg_key,seg_val)|
+        data.each {|tag,time,value,seg_key,seg_val,mode|
+          # TODO update_mode is not supported yet
           base_time = time / ROW_TIME_WINDOW
           metric_id = get_metric_id(db, tag, seg_key)
           segment_id = get_segment_id(db, seg_val) if seg_val

data/lib/fluent/plugin/{backend → backends}/stdout_backend.rb

@@ -18,10 +18,10 @@
 module Fluent::MetricSenseOutput::Backends
 
   class StdoutBackend < Fluent::MetricSenseOutput::Backend
-    Fluent::MetricSenseOutput::BACKENDS['stdout'] = self
+    Fluent::MetricSenseOutput.register_backend('stdout', self)
 
     def write(data)
-      data.each {|tag,time,value,seg_key,seg_val|
+      data.each {|tag,time,value,seg_key,seg_val,mode|
         if seg_key
           puts "#{Time.at(time)} #{tag}: #{value}"
         else

data/lib/fluent/plugin/out_metricsense.rb

@@ -23,28 +23,43 @@ module Fluent
 
     BACKENDS = {}
 
+    def self.register_backend(name, klass)
+      BACKENDS[name] = klass
+    end
+
+    module UpdateMode
+      ADD = 0
+      LATEST = 1
+    end
+
     class Backend
+      UpdateMode = MetricSenseOutput::UpdateMode
       include Configurable
-
       def start
       end
-
       def shutdown
       end
     end
 
-    backend_dir = "#{File.dirname(__FILE__)}/backend"
-    Dir.glob("#{backend_dir}/*_backend.rb") {|e|
-      require e
-    }
+    module Backends
+      backend_dir = "#{File.dirname(__FILE__)}/backends"
+      require "#{backend_dir}/librato_backend"
+      require "#{backend_dir}/rdb_tsdb_backend"
+      require "#{backend_dir}/stdout_backend"
+    end
 
-    config_param :segment_keys, :string, :default => nil
-    config_param :all_segment, :bool, :default => false
     config_param :value_key, :string, :default => 'value'
 
-    config_param :backend, :string
+    config_param :no_segment_keys, :bool, :default => false
+    config_param :only_segment_keys, :string, :default => nil
+    config_param :exclude_segment_keys, :string, :default => nil
+
+    config_param :update_mode_key, :string, :default => 'update_mode'
 
     config_param :remove_tag_prefix, :string, :default => nil
+    config_param :add_tag_prefix, :string, :default => nil
+
+    config_param :backend, :string
 
     def configure(conf)
       super
@@ -53,16 +68,14 @@ module Fluent
         @remove_tag_prefix = Regexp.new('^' + Regexp.escape(@remove_tag_prefix) + "\\.?")
       end
 
-      if conf.has_key?('all_segment') && conf['all_segment'].empty?
-        @all_segment = true
+      @no_segment_keys = (conf.has_key?('no_segment_keys') && (conf['no_segment_keys'].empty? || conf['no_segment_keys'] == 'true'))
+
+      if @only_segment_keys
+        @only_segment_keys = @only_segment_keys.strip.split(/\s*,\s*/)
       end
 
-      if @all_segment
-        @segment_keys = nil
-      elsif @segment_keys
-        @segment_keys = @segment_keys.strip.split(/\s*,\s*/)
-      else
-        @segment_keys = []
+      if @exclude_segment_keys
+        @exclude_segment_keys = @exclude_segment_keys.strip.split(/\s*,\s*/)
       end
 
       be = BACKENDS[@backend]
@@ -85,14 +98,27 @@ module Fluent
     end
 
     def format_stream(tag, es)
-      out = ''
+      # modify tag
       tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
+      tag = "#{add_tag_prefix}.#{tag}" if @add_tag_prefix
+
+      out = ''
       es.each do |time,record|
-        value = record[@value_key]
+        # dup record to modify
+        record = record.dup
 
-        fv = value.to_f
+        # get value
+        value = record.delete(@value_key)
+
+        # ignore record if value is invalid or 0
+        begin
+          fv = value.to_f
+        rescue
+          next
+        end
         next if fv == 0.0
 
+        # use integer if value.to_f == value.to_f.to_i
         iv = fv.to_i
         if iv.to_f == fv
           value = iv
@@ -100,35 +126,71 @@ module Fluent
           value = fv
         end
 
-        seg_keys = @segment_keys
-        unless seg_keys
-          seg_keys = record.keys
-          seg_keys.delete(@value_key)
+        # get update_mode key
+        update_mode = record.delete(@update_mode_key)
+        case update_mode
+        when "latest"
+          update_mode = UpdateMode::LATEST
+        else
+          # default is add
+          update_mode = UpdateMode::ADD
         end
 
-        segs = []
-        seg_keys.each {|seg_key|
-          if seg_val = record[seg_key]
-            segs << seg_key
-            segs << seg_val
+        # get segments
+        if @no_segment_keys
+          segments = {}
+        else
+          if @only_segment_keys
+            segments = {}
+            @only_segment_keys.each {|key|
+              if v = record[key]
+                segments[key] = v
+              end
+            }
+          else
+            segments = record
           end
-        }
+          if @exclude_segment_keys
+            @exclude_segment_keys.each {|key|
+              segments.delete(key)
+            }
+          end
+        end
 
-        [tag, time, value, segs].to_msgpack(out)
+        [tag, time, value, segments, update_mode].to_msgpack(out)
       end
+
       out
     end
 
-    class SumAggregator
+    class AddUpdater
       def initialize
         @value = 0
       end
+      attr_reader :value
 
       def add(value)
         @value += value
       end
 
+      def mode
+        UpdateMode::ADD
+      end
+    end
+
+    class LatestUpdater
+      def initialize
+        @value = 0
+      end
       attr_reader :value
+
+      def add(value)
+        @value += value
+      end
+
+      def mode
+        UpdateMode::LATEST
+      end
     end
 
     AggregationKey = Struct.new(:tag, :time, :seg_val, :seg_key)
@@ -137,21 +199,34 @@ module Fluent
       counters = {}
 
       # select sum(value) from chunk group by tag, time/60, seg_val, seg_key
-      chunk.msgpack_each {|tag,time,value,segs|
+      chunk.msgpack_each {|tag,time,value,segments,update_mode|
         time = time / 60 * 60
 
+        case update_mode
+        when UpdateMode::ADD
+          updater = AddUpdater
+        when UpdateMode::LATEST
+          updater = LatestUpdater
+        else  # default is AddUpdater
+          updater = AddUpdater
+        end
+
+        # total value
         ak = AggregationKey.new(tag, time, nil, nil)
-        (counters[ak] ||= SumAggregator.new).add(value)
+        (counters[ak] ||= updater.new).add(value)
+
+        # segmented values
+        segments = Hash[segments] if segments.is_a?(Array)  # for backward compat
 
-        segs.each_slice(2) {|seg_key,seg_val|
+        segments.each_pair {|seg_key,seg_val|
           ak = AggregationKey.new(tag, time, seg_val, seg_key)
-          (counters[ak] ||= SumAggregator.new).add(value)
+          (counters[ak] ||= updater.new).add(value)
         }
       }
 
       data = []
-      counters.each_pair {|ak,aggr|
-        data << [ak.tag, ak.time, aggr.value, ak.seg_key, ak.seg_val]
+      counters.each_pair {|ak,up|
+        data << [ak.tag, ak.time, up.value, ak.seg_key, ak.seg_val, up.mode]
       }
 
       @backend.write(data)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-metricsense
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-26 00:00:00.000000000 Z
+date: 2013-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -87,9 +87,9 @@ files:
 - Rakefile
 - VERSION
 - fluent-plugin-metricsense.gemspec
-- lib/fluent/plugin/backend/librato_backend.rb
-- lib/fluent/plugin/backend/rdb_tsdb_backend.rb
-- lib/fluent/plugin/backend/stdout_backend.rb
+- lib/fluent/plugin/backends/librato_backend.rb
+- lib/fluent/plugin/backends/rdb_tsdb_backend.rb
+- lib/fluent/plugin/backends/stdout_backend.rb
 - lib/fluent/plugin/out_metricsense.rb
 homepage: https://github.com/treasure-data/fluent-plugin-metricsense
 licenses: []
@@ -103,18 +103,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 143460017522358519
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 143460017522358519
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -122,3 +116,4 @@ signing_key:
 specification_version: 3
 summary: MetricSense - application metrics aggregation plugin for Fluentd
 test_files: []
+has_rdoc: false