fluent-plugin-mongokpi 0.0.0 → 0.0.1

data/README.md

@@ -1,24 +1,144 @@
 # Fluent::Plugin::Mongokpi
 
-Fluent BufferedOutput plugin: counting chunk, inserting counts to make kpi count on MongoDB
+## Description
+Fluent BufferedOutput plugin: counting chunk, inserting counts to MongoDB
 
 ## Installation
+    $ gem install fluent-plugin-mongokpi
 
-Add this line to your application's Gemfile:
+## Configuration
+config             | default         | info
+-------------------|-----------------|-----
+address            | localhost:27017 | address of mongodb (for replicaset, plz set like localhost:27017,localhost:27018,localhost:27019)
+db                 | kpidb           | db name of mongodb
+collection         | kpiColyyyymmdd  | collection name of mongodb (yyyymmdd will be replaced based on time)
+write_concern      |                 | mongodb connection option
+name               |                 | mongodb connection option
+read               |                 | mongodb connection option
+refresh_mode       |                 | mongodb connection option
+refresh_interval   |                 | mongodb connection option
+kpi_type           |                 | use 'access' for activating access count
+kpi_unit           | min             | time unit of count [min,hour,day]
+time_key           |                 | this time_key field or time of record is used
+time_format        |                 | (with time_key) the value of time_key will be parsed with this format
+count_key          |                 | used like group by (for multiple keys, plz set like site,host,instance)
+count_name         | count           | the count result will be set to this field
+response_threshold | 1000000         | (with kpi_type = access) used as threshold of response time
+f_code             | code            | (with kpi_type = access) field name of response code in the log records
+f_response_time    | response_time   | (with kpi_type = access) field name of response time in the log records
 
-    gem 'fluent-plugin-mongokpi'
+## MongoDB document
+field           | notification | info
+----------------|--------------|-----
+_id             |              | time + value of ${count_key}
+yyyymmdd        |              | 
+hh              |              | (with kpi_unit = min or hour)
+mm              |              | (with kpi_unit = min)
+${count_key}    |              | 
+${count_name}   |              | count
+count1xx        |              | (with kpi_type = access) count of ${f_code} = 1XX
+count2xx        |              | (with kpi_type = access) count of ${f_code} = 2XX
+count3xx        |              | (with kpi_type = access) count of ${f_code} = 3XX
+count4xx        |              | (with kpi_type = access) count of ${f_code} = 4XX
+count5xx        |              | (with kpi_type = access) count of ${f_code} = 5XX
+responseTimeAve | `m(_ _)m`    | (with kpi_type = access) average of ${f_response_time}
+responseTimeMax | `m(_ _)m`    | (with kpi_type = access) maximum of ${f_response_time}
+responseTimeMin | `m(_ _)m`    | (with kpi_type = access) minimum of ${f_response_time}
+responseTimeSum |              | (with kpi_type = access) sum of ${f_response_time}
+countOver       |              | (with kpi_type = access) count of the access: ${f_response_time} >= ${response_threshold}
+okRatio         | `m(_ _)m`    | (with kpi_type = access) (count - countOver) / count
+counter[60]     |              | (with kpi_type = access & kpy_unit = min) qps for each second
+qpsAve          | `m(_ _)m`    | (with kpi_type = access & kpy_unit = min) average of qps
+qpsMax          | `m(_ _)m`    | (with kpi_type = access & kpy_unit = min) maximum of qps
+qpsMin          | `m(_ _)m`    | (with kpi_type = access & kpy_unit = min) minimum of qps
 
-And then execute:
+`m(_ _)m` : these parameters are updated by non-atmic queries, then not reliable in multi processing
 
-    $ bundle
+## Samples
+### sample: access log (simple)
+config
 
-Or install it yourself as:
+    <match ***>
+      type mongokpi
+      kpi_type access
+      kpi_unit hourly
+      count_key site
+      time_key request_time
+      time_format %d/%b/%Y:%H:%M:%S %Z
+    </match>
 
-    $ gem install fluent-plugin-mongokpi
+log
+
+    {"request_time":"25/Nov/2013:20:03:02 +0700","code":"100","response_time":"1000","site":"git"}
+    {"request_time":"25/Nov/2013:20:03:03 +0700","code":"200","response_time":"2000","site":"git"}
+    {"request_time":"25/Nov/2013:20:03:03 +0700","code":"300","response_time":"3000","site":"git"}
+    {"request_time":"25/Nov/2013:20:03:03 +0700","code":"404","response_time":"4000","site":"blog"}
+    {"request_time":"25/Nov/2013:20:03:03 +0700","code":"500","response_time":"5000","site":"blog"}
+
+MongoDB
+
+    db.kpiCol20131125.find()
+    { "_id" : "git201311252003",   "count" : 3, "count1xx" : 1, "count2xx" : 1, "count3xx" : 1, "count4xx" : 0, "count5xx" : 0, "countOver" : 0,
+      "hh" : "20", "okRatio" : 1, "responseTimeAve" : 2000, "responseTimeMax" : 3000, "responseTimeMin" : 1000, "responseTimeSum" : 6000, "site" : "git", "yyyymmdd" : "20131125"}
+    { "_id" : "blog201311252003",  "count" : 2, "count1xx" : 0, "count2xx" : 0, "count3xx" : 0, "count4xx" : 4, "count5xx" : 1, "countOver" : 0,
+      "hh" : "20", "okRatio" : 1, "responseTimeAve" : 4500, "responseTimeMax" : 5000, "responseTimeMin" : 4000, "responseTimeSum" : 9000, "site" : "blog", "yyyymmdd" : "20131125"}
+    { "_id" : "total201311252003", "count" : 5, "count1xx" : 0, "count2xx" : 0, "count3xx" : 0, "count4xx" : 0, "count5xx" : 0, "countOver" : 0,
+      "hh" : "20", "okRatio" : 1, "responseTimeAve" : 3000, "responseTimeMax" : 5000, "responseTimeMin" : 1000, "responseTimeSum" : 15000, "site" : "total", "yyyymmdd" : "20131125" }
+
+### sample: access log (detail)
+config
+
+    <match ***>
+      type mongokpi
+      kpi_type access
+      count_key site
+      time_key request_time
+      time_format %d/%b/%Y:%H:%M:%S %Z
+    </match>
+
+log
 
-## Usage
+    {"request_time":"25/Nov/2013:20:03:02 +0700","code":"100","response_time":"1000","site":"git"}
+    {"request_time":"25/Nov/2013:20:03:03 +0700","code":"200","response_time":"2000","site":"git"}
+    {"request_time":"25/Nov/2013:20:03:03 +0700","code":"300","response_time":"3000","site":"git"}
+    {"request_time":"25/Nov/2013:20:03:03 +0700","code":"404","response_time":"4000","site":"blog"}
+    {"request_time":"25/Nov/2013:20:03:03 +0700","code":"500","response_time":"5000","site":"blog"}
 
-    m(_ _)m
+MongoDB
+
+    db.kpiCol20131125.find()
+    { "_id" : "git201311252003",   "count" : 3, "count1xx" : 1, "count2xx" : 1, "count3xx" : 1, "count4xx" : 0, "count5xx" : 0, "countOver" : 0,
+      "counter" : [ 0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
+      "hh" : "20", "mm" : "03", "okRatio" : 1, "qpsAve" : 0, "qpsMax" : 2, "qpsMin" : 0, "responseTimeAve" : 2000, "responseTimeMax" : 3000, "responseTimeMin" : 1000, "responseTimeSum" : 6000, "site" : "git", "yyyymmdd" : "20131125"}
+    { "_id" : "blog201311252003",  "count" : 2, "count1xx" : 0, "count2xx" : 0, "count3xx" : 0, "count4xx" : 4, "count5xx" : 1, "countOver" : 0,
+      "counter" : [ 0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
+      "hh" : "20", "mm" : "03", "okRatio" : 1, "qpsAve" : 0, "qpsMax" : 2, "qpsMin" : 0, "responseTimeAve" : 4500, "responseTimeMax" : 5000, "responseTimeMin" : 4000, "responseTimeSum" : 9000, "site" : "blog", "yyyymmdd" : "20131125"}
+    { "_id" : "total201311252003", "count" : 5, "count1xx" : 0, "count2xx" : 0, "count3xx" : 0, "count4xx" : 0, "count5xx" : 0, "countOver" : 0,
+      "counter" : [ 0,0,1,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
+      "hh" : "20", "mm" : "03", "okRatio" : 1, "qpsAve" : 0, "qpsMax" : 4, "qpsMin" : 0, "responseTimeAve" : 3000, "responseTimeMax" : 5000, "responseTimeMin" : 1000, "responseTimeSum" : 15000, "site" : "total", "yyyymmdd" : "20131125" }
+
+### sample: app log
+config
+
+    <match ***>
+      type mongokpi
+      count_key site
+    </match>
+
+log
+
+    {"gerogero":"Let's get Rocking!","site":"git"}
+    {"gerogero":"Let's get Rocking!","site":"git"}
+    {"gerogero":"Let's get Rocking!","site":"git"}
+    {"gerogero":"Let's get Rocking!","site":"blog"}
+    {"gerogero":"Let's get Rocking!","site":"blog"}
+
+MongoDB
+
+    db.kpiCol20131125.find()
+    { "_id" : "git201311252248",  "count" : 3, "hh" : "22", "mm" : "48", "site" : "git", "yyyymmdd" : "20131125" }
+    { "_id" : "blog201311252248", "count" : 2, "hh" : "22", "mm" : "48", "site" : "blog", "yyyymmdd" : "20131125" }
+    { "_id" : "total201311252248","count" : 5, "hh" : "22", "mm" : "48", "site" : "total", "yyyymmdd" : "20131125" }
 
 ## Contributing
 
@@ -28,3 +148,4 @@ Or install it yourself as:
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
 
+

data/fluent-plugin-mongokpi.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mongokpi"
-  spec.version       = "0.0.0"
+  spec.version       = "0.0.1"
   spec.authors       = ["Hatayama Hideharu"]
   spec.email         = ["h.hiddy@gmail.com"]
   spec.description   = %q{Fluent BufferedOutput plugin: counting chunk, inserting counts to make kpi count on MongoDB}

data/lib/fluent/plugin/out_mongokpi.rb

@@ -4,6 +4,13 @@ module Fluent
     # which is used in the configuration file.
     Plugin.register_output('mongokpi', self)
     
+    UNIT_DAYS = ['d', 'day', 'dayly']
+    UNIT_HOURS = ['h', 'hour', 'hourly']
+    UNIT_MINUTES = ['m', 'min', 'minute']
+    DAY = 'day'
+    HOUR = 'hour'
+    MIN = 'min'
+    
     # Mongo client settings
     config_param :address,    :string, :default => 'localhost:27017'
     config_param :db,         :string, :default => 'kpidb'
@@ -17,18 +24,18 @@ module Fluent
     # collection options
     config_param :capped_size, :integer, :default => 0
     config_param :capped_max,  :integer, :default => 0
-    # KPI count options
+    # count options
+    config_param :kpi_type,    :string,  :default => nil # use 'access' for activating access count.
+    config_param :kpi_unit,    :string,  :default => 'min'
     config_param :time_key,    :string, :default => nil
     config_param :time_format, :string, :default => nil
     config_param :count_key,   :string, :default => 'none'
     config_param :count_name,  :string, :default => 'count'
     # access count options
-    config_param :kpi_type, :string, :default => nil # use 'access' for activating access count.
-    config_param :responseThreshold, :integer, :default => 1000000
-    config_param :f_code,          :string, :default => 'code'
-    config_param :f_response_time, :string, :default => 'response_time'
+    config_param :response_threshold, :integer, :default => 1000000
+    config_param :f_code,             :string, :default => 'code'
+    config_param :f_response_time,    :string, :default => 'response_time'
 
-    
     attr_reader :collections_opts, :connection_opts
 
     def initialize
@@ -43,6 +50,7 @@ module Fluent
     # If the configuration is invalid, raise Fluent::ConfigError.
     def configure(conf)
       super
+      @kpi_unit = get_kpi_unit(@kpi_unit)
       @connection_opts = {}
       @connection_opts[:w] = @write_concern unless @write_concern.nil?
       @connection_opts[:name] = @name unless @name.nil?
@@ -52,12 +60,20 @@ module Fluent
       @collections_opts = {}
       if @capped_size > 0
         @collections_opts[:capped] = true
-        @collections_opts[:size] = Config.size_value(conf['capped_size'])
-        @collections_opts[:max] = Config.size_value(conf['capped_max']) if @capped_max
+        @collections_opts[:size] = @capped_size
+        @collections_opts[:max] = @capped_max if @capped_max
       else
         @collections_opts[:capped] = false
       end
     end
+    
+    def get_kpi_unit(kpi_unit)
+      case kpi_unit
+        when *UNIT_DAYS  then return DAY
+        when *UNIT_HOURS then return HOUR
+        else                  return MIN
+      end
+    end
 
     # This method is called when starting.
     # Open sockets or files here.
@@ -99,8 +115,8 @@ module Fluent
     #  }
     #end
     def write(chunk)
-      doc_hash = get_insert_doc_hash(@kpi_type, chunk, @time_key, @time_format, @count_key, @count_name)
-      insert(@kpi_type, @collection, @count_key, @count_name, doc_hash)
+      doc_hash = get_insert_doc_hash(@kpi_type, @kpi_unit, @time_key, @time_format, @count_key, @count_name, chunk)
+      insert(@kpi_type, @kpi_unit, @collection, @count_key, @count_name, doc_hash)
     end
     
     def get_client(address, connection_opts)
@@ -130,14 +146,14 @@ module Fluent
     end
     
     def convert_collection_name(collection_name, yyyymmdd) 
-      return collection_name.sub('yyyymmdd', yyyymmdd)
+      return collection_name.sub('yyyymmdd', yyyymmdd.to_s)
     end
 
     def get_collection_from_db(client, db_name, collection_name, collections_opts)
       return client.db(db_name).collection(collection_name, @collections_opts)
     end
     
-    def get_insert_doc_hash(kpi_type, chunk, time_key, time_format, count_key, count_name)
+    def get_insert_doc_hash(kpi_type, kpi_unit, time_key, time_format, count_key, count_name, chunk)
       hash_counter = {}
       chunk.msgpack_each { |tag, time, record|
         $log.debug record
@@ -148,26 +164,40 @@ module Fluent
         if 'none' != count_key
           count_key_value = ''
           count_key.split(',').each { |x| count_key_value += record[x].to_s }
-          key_str = count_key_value + tmp_time.strftime('%Y%m%d%H%M')
+          key_str = count_key_value + get_time_key_value(kpi_unit, tmp_time)
           doc = hash_counter.key?(key_str) ? hash_counter[key_str]
-                  : get_doc(kpi_type, count_key, count_key_value, count_name, tmp_time)
-          hash_counter[key_str] = count_up(kpi_type, doc, record, count_name, tmp_time)
+                  : get_doc(kpi_type, kpi_unit, count_key, count_key_value, count_name, tmp_time)
+          hash_counter[key_str] = count_up(kpi_type, kpi_unit, doc, record, count_name, tmp_time)
         end
         # total
-        total_key_str = 'total' + tmp_time.strftime('%Y%m%d%H%M')
+        total_key_str = 'total' + get_time_key_value(kpi_unit, tmp_time)
         total = hash_counter.key?(total_key_str) ? hash_counter[total_key_str]
-                  : get_doc(kpi_type, count_key, 'total', count_name, tmp_time)
-        hash_counter[total_key_str] = count_up(kpi_type, total, record, count_name, tmp_time)
+                  : get_doc(kpi_type, kpi_unit, count_key, 'total', count_name, tmp_time)
+        hash_counter[total_key_str] = count_up(kpi_type, kpi_unit, total, record, count_name, tmp_time)
       }
       return hash_counter
     end
     
-    def get_doc(kpi_type, count_key, count_key_value, count_name, time)
+    def get_time_key_value(kpi_unit, time)
+      case kpi_unit
+        when DAY  then return time.strftime('%Y%m%d')
+        when HOUR then return time.strftime('%Y%m%d%H')
+        else           return time.strftime('%Y%m%d%H%M')
+      end
+    end
+    
+    def get_doc(kpi_type, kpi_unit, count_key, count_key_value, count_name, time)
       doc = {}
       doc[count_key] = count_key_value
-      doc['yyyymmdd'] = time.strftime('%Y%m%d')
-      doc['hh'] = time.strftime('%H')
-      doc['mm'] = time.strftime('%M')
+      time_field_hash = {}
+      time_field_hash['yyyymmdd'] = time.strftime('%Y%m%d').to_i
+      if HOUR == kpi_unit
+        time_field_hash['hh'] = time.strftime('%H').to_i
+      elsif MIN == kpi_unit
+        time_field_hash['hh'] = time.strftime('%H').to_i
+        time_field_hash['mm'] = time.strftime('%M').to_i
+      end
+      doc['time_field_hash'] = time_field_hash
       doc[count_name] = 0
       if 'access' == kpi_type
         doc['countOver'] = 0
@@ -178,22 +208,24 @@ module Fluent
         doc['count5xx'] = 0
         doc['responseTimeAve'] = 0
         doc['responseTimeMax'] = 0
-        doc['responseTimeMin'] = 100000000.00
+        doc['responseTimeMin'] = 100000000
         doc['responseTimeSum'] = 0
-        doc['qpsAve'] = 0
-        doc['qpsMax'] = 0
-        doc['qpsMin'] = 100000000
-        doc['okRatio'] = 0.00
-        doc['counter'] = Array.new(60, 0)
+        doc['okRatio'] = 0.0000
+        if MIN == kpi_unit
+          doc['qpsAve'] = 0
+          doc['qpsMax'] = 0
+          doc['qpsMin'] = 100000000
+          doc['counter'] = Array.new(60, 0)
+        end
       end
       return doc
     end
     
-    def count_up(kpi_type, doc, record, count_name, time)
+    def count_up(kpi_type, kpi_unit, doc, record, count_name, time)
       doc[count_name] += 1
       if 'access' == kpi_type
         response_time = record.key?(@f_response_time) ? record[@f_response_time].to_i : 0
-        if response_time > @responseThreshold
+        if response_time >= @response_threshold
           doc['countOver'] += 1
         end
         case record[@f_code].to_i / 100
@@ -210,140 +242,88 @@ module Fluent
           doc['responseTimeMin'] = response_time
         end
         doc['responseTimeSum'] += response_time
-        doc['counter'][time.strftime('%S').to_i] += 1
+        doc['counter'][time.strftime('%S').to_i] += 1 if MIN == kpi_unit
       end
       return doc
     end
     
-    def insert(kpi_type, collection_name, count_key, count_name, doc_hash)
+    # 2.5 or less
+    # http://stackoverflow.com/questions/8508663/calculate-max-value-in-an-atomic-findandmodify-operation
+    # TODO improve for Mongo 2.6
+    # $min, $max field update operators
+    # https://jira.mongodb.org/browse/SERVER-1534
+    # https://jira.mongodb.org/browse/DOCS-2012
+    def insert(kpi_type, kpi_unit, collection_name, count_key, count_name, doc_hash)
       begin
         doc_hash.each { |key, doc|
-          $log.debug doc
-          collection = get_collection(collection_name, doc['yyyymmdd'])
-          # 2.5 or less
-          # http://stackoverflow.com/questions/8508663/calculate-max-value-in-an-atomic-findandmodify-operation
-          # TODO improve for Mongo 2.6
-          # $min, $max field update operators
-          # https://jira.mongodb.org/browse/SERVER-1534
-          # https://jira.mongodb.org/browse/DOCS-2012
+          $log.debug "key: #{key}, doc: #{doc}"
+          collection = get_collection(collection_name, doc['time_field_hash']['yyyymmdd'])
+          select_hash = doc['time_field_hash']
+          select_hash['_id'] = key
+          select_hash[count_key] = doc[count_key]
           if 'access' == kpi_type
+            # initialize the target doc
+            # Without this, "0" in "counter.0' can be regarded as an child element, not as an array element.
+            if MIN == kpi_unit
+              collection.update(
+                select_hash,
+                {'$setOnInsert' => {'counter' => Array.new(60, 0)}},
+                {:upsert => true}
+              )
+            end
+            # main update
+            increment_hash = {}
+            increment_hash[count_name] = doc[count_name]
+            increment_hash['countOver'] = doc['countOver']
+            increment_hash['count1xx'] = doc['count1xx']
+            increment_hash['count2xx'] = doc['count2xx']
+            increment_hash['count3xx'] = doc['count3xx']
+            increment_hash['count4xx'] = doc['count4xx']
+            increment_hash['count5xx'] = doc['count5xx']
+            if MIN == kpi_unit
+              for sec in 0..59
+                increment_hash['counter.' + sec.to_s] = doc['counter'][sec]
+              end            
+            end
+            increment_hash['responseTimeSum'] = doc['responseTimeSum']
             collection.update(
-              {'_id' => key, count_key => doc[count_key],
-                'yyyymmdd' => doc['yyyymmdd'], 'hh' => doc['hh'], 'mm' => doc['mm']},
-              {'$setOnInsert' => {'counter' => Array.new(60, 0)}},
-              {:upsert => true}
-            )
-            collection.update(
-              {'_id' => key, count_key => doc[count_key],
-                'yyyymmdd' => doc['yyyymmdd'], 'hh' => doc['hh'], 'mm' => doc['mm']},
-              {'$inc' => {
-                count_name => doc[count_name],
-                'countOver' => doc['countOver'],
-                'count1xx' => doc['count1xx'],
-                'count2xx' => doc['count2xx'],
-                'count3xx' => doc['count3xx'],
-                'count4xx' => doc['count4xx'],
-                'count5xx' => doc['count5xx'],
-                'responseTimeSum' => doc['responseTimeSum'],
-                'counter.0' => doc['counter'][0],
-                'counter.1' => doc['counter'][1],
-                'counter.2' => doc['counter'][2],
-                'counter.3' => doc['counter'][3],
-                'counter.4' => doc['counter'][4],
-                'counter.5' => doc['counter'][5],
-                'counter.6' => doc['counter'][6],
-                'counter.7' => doc['counter'][7],
-                'counter.8' => doc['counter'][8],
-                'counter.9' => doc['counter'][9],
-                'counter.10' => doc['counter'][10],
-                'counter.11' => doc['counter'][11],
-                'counter.12' => doc['counter'][12],
-                'counter.13' => doc['counter'][13],
-                'counter.14' => doc['counter'][14],
-                'counter.15' => doc['counter'][15],
-                'counter.16' => doc['counter'][16],
-                'counter.17' => doc['counter'][17],
-                'counter.18' => doc['counter'][18],
-                'counter.19' => doc['counter'][19],
-                'counter.20' => doc['counter'][20],
-                'counter.21' => doc['counter'][21],
-                'counter.22' => doc['counter'][22],
-                'counter.23' => doc['counter'][23],
-                'counter.24' => doc['counter'][24],
-                'counter.25' => doc['counter'][25],
-                'counter.26' => doc['counter'][26],
-                'counter.27' => doc['counter'][27],
-                'counter.28' => doc['counter'][28],
-                'counter.29' => doc['counter'][29],
-                'counter.30' => doc['counter'][30],
-                'counter.31' => doc['counter'][31],
-                'counter.32' => doc['counter'][32],
-                'counter.33' => doc['counter'][33],
-                'counter.34' => doc['counter'][34],
-                'counter.35' => doc['counter'][35],
-                'counter.36' => doc['counter'][36],
-                'counter.37' => doc['counter'][37],
-                'counter.38' => doc['counter'][38],
-                'counter.39' => doc['counter'][39],
-                'counter.40' => doc['counter'][40],
-                'counter.41' => doc['counter'][41],
-                'counter.42' => doc['counter'][42],
-                'counter.43' => doc['counter'][43],
-                'counter.44' => doc['counter'][44],
-                'counter.45' => doc['counter'][45],
-                'counter.46' => doc['counter'][46],
-                'counter.47' => doc['counter'][47],
-                'counter.48' => doc['counter'][48],
-                'counter.49' => doc['counter'][49],
-                'counter.50' => doc['counter'][50],
-                'counter.51' => doc['counter'][51],
-                'counter.52' => doc['counter'][52],
-                'counter.53' => doc['counter'][53],
-                'counter.54' => doc['counter'][54],
-                'counter.55' => doc['counter'][55],
-                'counter.56' => doc['counter'][56],
-                'counter.57' => doc['counter'][57],
-                'counter.58' => doc['counter'][58],
-                'counter.59' => doc['counter'][59]
-              }},
+              select_hash,
+              {'$inc' => increment_hash},
               {:upsert => true}
             )
-            updated_doc_array = collection.find({'_id' => key}).to_a
-            $log.debug updated_doc_array
-            continue if updated_doc_array.nil?
-            updated_doc = updated_doc_array[0]
-            response_time_ave = updated_doc['responseTimeSum'] / doc[count_name]
+            # add supplemental fields using existing data
+            # NOTICE: this operation is not atmic, then the field value can be wrong in distributed processing.
+            updated_result = collection.find({'_id' => key})
+            if updated_result.nil?
+              $log.info "there is no updated result for the key: #{key}" if updated_result.nil?
+              continue
+            end
+            updated_doc = updated_result.to_a[0]
+            set_hash = {}
+            set_hash['responseTimeAve'] = updated_doc['responseTimeSum'] / doc[count_name]
             if !updated_doc['responseTimeMax'].nil? && updated_doc['responseTimeMax'] > doc['responseTimeMax']
-              response_time_max = updated_doc['responseTimeMax']
+              set_hash['responseTimeMax'] = updated_doc['responseTimeMax']
             else
-              response_time_max = doc['responseTimeMax']
+              set_hash['responseTimeMax'] = doc['responseTimeMax']
             end
             if !updated_doc['responseTimeMin'].nil? && updated_doc['responseTimeMin'] < doc['responseTimeMin']
-              response_time_min = updated_doc['responseTimeMin']
+              set_hash['responseTimeMin'] = updated_doc['responseTimeMin']
             else
-              response_time_min = doc['responseTimeMin']
+              set_hash['responseTimeMin'] = doc['responseTimeMin']
+            end
+            set_hash['okRatio'] = ((updated_doc[count_name] - updated_doc['countOver']).to_f / updated_doc[count_name]).round(4)
+            if MIN == kpi_unit
+              set_hash['qpsAve'] = (updated_doc['counter'].inject(0.0){|r,i| r+=i } / updated_doc['counter'].size).round
+              set_hash['qpsMax'] = updated_doc['counter'].max
+              set_hash['qpsMin'] = updated_doc['counter'].min
             end
-            qps_ave = (updated_doc['counter'].inject(0.0){|r,i| r+=i } / updated_doc['counter'].size).round
-            qps_max = updated_doc['counter'].max
-            qps_min = updated_doc['counter'].min
-            ok_ratio = ((updated_doc[count_name] - updated_doc['countOver']).to_f / updated_doc[count_name]).round(4)
             collection.update(
-              {'_id' => key, count_key => doc[count_key],
-                'yyyymmdd' => doc['yyyymmdd'], 'hh' => doc['hh'], 'mm' => doc['mm']},
-              { '$set' => {
-              'responseTimeAve' => response_time_ave,
-              'responseTimeMax' => response_time_max,
-              'responseTimeMin' => response_time_min,
-              'qpsAve' => qps_ave,
-              'qpsMax' => qps_max,
-              'qpsMin' => qps_min,
-              'okRatio' => ok_ratio
-              }}
+              select_hash,
+              { '$set' => set_hash}
             )
           else
             collection.update(
-              {'_id' => key, count_key => doc[count_key],
-                'yyyymmdd' => doc['yyyymmdd'], 'hh' => doc['hh'], 'mm' => doc['mm']},
+              select_hash,
               {'$inc' => {count_name => doc[count_name]}},
               {:upsert => true}
             )

data/test/fluent/plugin/out_mongokpi_test.rb

@@ -57,7 +57,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
 
   def test_format
     d = create_driver
-    time = Time.parse("2112-09-03 01:23:45 UTC").to_i
+    time = Time.parse("2112-09-03 01:23:45 UTC")
     d.emit({"gerogero" => "Let's get Rocking!", "site" => "yapoo"}, time)
     d.emit({"gerogero" => "Let's get Rocking!", "site" => "geegero"}, time)
     d.expect_format "\x93\xABdefault_tag\xCF\x00\x00\x00\x01\f[\xC8\xA1\x82\xA8gerogero\xB2Let's get Rocking!\xA4site\xA5yapoo\x93\xABdefault_tag\xCF\x00\x00\x00\x01\f[\xC8\xA1\x82\xA8gerogero\xB2Let's get Rocking!\xA4site\xA7geegero"
@@ -88,3 +88,4 @@ class DataCounterOutputTest < Test::Unit::TestCase
 end
 
 
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongokpi
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-27 00:00:00.000000000 Z
+date: 2013-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler