fluent-plugin-metricsense 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ebe8e81e18307cb99946d1440e937b23426a7fa3
-  data.tar.gz: f8ea8e769f2b91ee8d8f738f19698c88d4f5803c
+  metadata.gz: 86baf51deb386cc4708139e311f0564a70d6232c
+  data.tar.gz: 83c96af6a0b6f4b3bf27742a3e7659dfa499c43d
 SHA512:
-  metadata.gz: ef1fdac5be54fce8d388345123e65639801901898f47561b0026d2f6f1a514280bac5073601b5819c918e04dee0ccd01db4a0536c2ee87ca58334125774640b4
-  data.tar.gz: d73a48b3091533eae618bde06f3cb9166e781854c9e06853953daeddcca64d5fb58966bba09f74c7919de84790ba83adbfb4377bdf83940c9f8f47e18f924d08
+  metadata.gz: 24e59e73c77cb9a2a953edbafd7c4f5a56b25bef749a00e1ca8e152edf29d39f60ac2c7ad4d8cf22afffe27570fe70f496d6c47554214c58525c64bc47f72817
+  data.tar.gz: f899dea67543ffe32bee27f3e72b75585579100da78dfbece9bc5bc990ec59f33e7cb7d6b2f51a482f990b808d6da5ff2237d8dcae32a959fdd0ceec0157942a

data/Rakefile

@@ -1,13 +1,13 @@
 require 'bundler'
 Bundler::GemHelper.install_tasks
 
-#require 'rake/testtask'
-#
-#Rake::TestTask.new(:test) do |test|
-#  test.libs << 'lib' << 'test'
-#  test.test_files = FileList['test/plugin/*.rb']
-#  test.verbose = true
-#end
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
 
 #task :coverage do |t|
 #  ENV['SIMPLE_COV'] = '1'

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.3.2

data/fluent-plugin-metricsense.gemspec

@@ -20,5 +20,6 @@ Gem::Specification.new do |s|
   s.add_dependency "dogapi"
   s.add_development_dependency "rake", ">= 0.8.7"
   s.add_development_dependency 'bundler', ['>= 1.0.0']
-  s.add_development_dependency "simplecov", ">= 0.5.4"
+  s.add_development_dependency "test-unit", ">= 3.1.1"
+  s.add_development_dependency "simplecov", ">= 0.10.0"
 end

data/lib/fluent/plugin/out_metricsense.rb

@@ -262,30 +262,34 @@ module Fluent
 
       # select sum(value) from chunk group by tag, time/60, seg_val, seg_key
       chunk.msgpack_each {|tag,time,value,segments,update_mode|
-        time = time / @aggregate_interval * @aggregate_interval
+        begin
+          time = time / @aggregate_interval * @aggregate_interval
+
+          case update_mode
+          when UpdateMode::ADD
+            updater = AddUpdater
+          when UpdateMode::MAX
+            updater = MaxUpdater
+          when UpdateMode::AVERAGE # AVERAGE uses MaxUpdater and calculate average on server-side aggregation
+            updater = AverageUpdater
+          when UpdateMode::COUNT
+            updater = CountUpdater
+          else  # default is AddUpdater
+            updater = AddUpdater
+          end
 
-        case update_mode
-        when UpdateMode::ADD
-          updater = AddUpdater
-        when UpdateMode::MAX
-          updater = MaxUpdater
-        when UpdateMode::AVERAGE # AVERAGE uses MaxUpdater and calculate average on server-side aggregation
-          updater = AverageUpdater
-        when UpdateMode::COUNT
-          updater = CountUpdater
-        else  # default is AddUpdater
-          updater = AddUpdater
+          # simple values
+          ak = AggregationKey.new(tag, time, nil, nil)
+          (simple_counters[ak] ||= updater.new).add(value)
+
+          # segmented values
+          segments.each_pair {|seg_key,seg_val|
+            ak = AggregationKey.new(tag, time, seg_val, seg_key)
+            (segmented_counters[ak] ||= updater.new).add(value)
+          }
+        rescue StandardError => e
+          log.warn("ignoring broken chunk: #{e.inspect} - " + [tag, time, value, segments, update_mode].inspect)
         end
-
-        # simple values
-        ak = AggregationKey.new(tag, time, nil, nil)
-        (simple_counters[ak] ||= updater.new).add(value)
-
-        # segmented values
-        segments.each_pair {|seg_key,seg_val|
-          ak = AggregationKey.new(tag, time, seg_val, seg_key)
-          (segmented_counters[ak] ||= updater.new).add(value)
-        }
       }
 
       counters = segmented_counters

data/test/plugin/test_out_metricsense.rb

@@ -0,0 +1,56 @@
+require 'fluent/test'
+require 'fluent/plugin/out_metricsense'
+
+class MetricsenseOutputTest < Test::Unit::TestCase
+  class TestBackend < Fluent::MetricSenseOutput::Backend
+    Fluent::MetricSenseOutput.register_backend('test', self)
+
+    @@data = []
+
+    def self.data
+      @@data
+    end
+
+    def write(data)
+      @@data << data
+    end
+  end
+
+  CONFIG = %Q[
+    backend test
+  ]
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MetricSenseOutput.new).configure(conf)
+  end
+
+  def test_emit
+    now = Time.now.to_i
+    d = create_driver(CONFIG)
+    data = {'value' => 1, 'user_id' => 23456, 'path' => '/auth/login'}
+    d.emit(data, now)
+    d.run
+
+    t = now / 60 * 60
+    TestBackend.data.each do |written|
+      assert_equal ['test', t, 1, 'user_id', 23456, 0], written.shift
+      assert_equal ['test', t, 1, 'path', "/auth/login", 0], written.shift
+      assert_equal ['test', t, 1, nil, nil, 0], written.shift
+    end
+  end
+
+  # server restart can cause broken buffer and broken chunk.
+  # out_metricsense ignore the chunk with warning log
+  def test_skip_broken_chunk
+    d = create_driver(CONFIG)
+    data = {'value' => 1, 'user_id' => 23456, 'path' => '/auth/login'}
+    d.emit(data, data)
+    d.run
+
+    assert_equal [], TestBackend.data.first
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-metricsense
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-06 00:00:00.000000000 Z
+date: 2017-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -66,20 +66,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.1
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.10.0
 description: MetricSense - application metrics aggregation plugin for Fluentd
 email: sf@treasure-data.com
 executables: []
@@ -98,6 +112,7 @@ files:
 - lib/fluent/plugin/backends/rdb_tsdb_backend.rb
 - lib/fluent/plugin/backends/stdout_backend.rb
 - lib/fluent/plugin/out_metricsense.rb
+- test/plugin/test_out_metricsense.rb
 homepage: https://github.com/treasure-data/fluent-plugin-metricsense
 licenses:
 - Apache-2.0
@@ -118,8 +133,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: MetricSense - application metrics aggregation plugin for Fluentd
-test_files: []
+test_files:
+- test/plugin/test_out_metricsense.rb