fluent-plugin-metricsense 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ebe8e81e18307cb99946d1440e937b23426a7fa3
-  data.tar.gz: f8ea8e769f2b91ee8d8f738f19698c88d4f5803c
+SHA256:
+  metadata.gz: 7371bfe5a22e6d5bcad59eb8519ee42b74ca08f563a914cb3e81c14d0920599a
+  data.tar.gz: 66f6e9c5c425419c6f8aad48a8a2fc7d2eadddfed2777341e164a692d6a4b48b
 SHA512:
-  metadata.gz: ef1fdac5be54fce8d388345123e65639801901898f47561b0026d2f6f1a514280bac5073601b5819c918e04dee0ccd01db4a0536c2ee87ca58334125774640b4
-  data.tar.gz: d73a48b3091533eae618bde06f3cb9166e781854c9e06853953daeddcca64d5fb58966bba09f74c7919de84790ba83adbfb4377bdf83940c9f8f47e18f924d08
+  metadata.gz: 817364fce002c27ad75bea81cafc6f002253688c7feb39077ba5927f2b8611d16e6cd65880f4961f1fafdca6b4898fb88507bd7691242343f953fcd2fb0e6833
+  data.tar.gz: 896b4d773cda010e95ab7d617945f63ee44707213c1c2cecb65bbdc938842bb5b541caa4afb4ceee19d5fd6d18a450742b5e8cd656ef45e4e4b69dbf7517fd77

data/Rakefile

@@ -1,13 +1,13 @@
 require 'bundler'
 Bundler::GemHelper.install_tasks
 
-#require 'rake/testtask'
-#
-#Rake::TestTask.new(:test) do |test|
-#  test.libs << 'lib' << 'test'
-#  test.test_files = FileList['test/plugin/*.rb']
-#  test.verbose = true
-#end
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
 
 #task :coverage do |t|
 #  ENV['SIMPLE_COV'] = '1'

data/SECURITY.md

@@ -0,0 +1,7 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Treasure Data values the security of its customers and is committed to ensuring that the systems and products are secure. We invite all bug bounty researchers to join our efforts in identifying and reporting vulnerabilities in our systems.
+
+Submit your findings to our dedicated bug bounty email address [vulnerabilities@treasuredata.com](mailto:vulnerabilities@treasuredata.com) and help us keep Treasure Data secure. Let’s work together to make the Internet a safer place!

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.4.0

data/fluent-plugin-metricsense.gemspec

@@ -16,9 +16,10 @@ Gem::Specification.new do |s|
   s.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
 
-  s.add_dependency "fluentd", ">= 0.10.6"
+  s.add_dependency "fluentd", [">= 0.14.0", "< 2"]
   s.add_dependency "dogapi"
   s.add_development_dependency "rake", ">= 0.8.7"
   s.add_development_dependency 'bundler', ['>= 1.0.0']
-  s.add_development_dependency "simplecov", ">= 0.5.4"
+  s.add_development_dependency "test-unit", ">= 3.1.1"
+  s.add_development_dependency "simplecov", ">= 0.10.0"
 end

data/lib/fluent/plugin/out_metricsense.rb

@@ -18,9 +18,19 @@
 #
 module Fluent
 
-  class MetricSenseOutput < Fluent::BufferedOutput
+  class MetricSenseOutput < Fluent::Plugin::Output
     Fluent::Plugin.register_output('metricsense', self)
 
+    helpers :compat_parameters
+
+    def formatted_to_msgpack_binary?
+      true
+    end
+
+    def compat_parameters_default_chunk_key
+      ""
+    end
+
     BACKENDS = {}
 
     def self.register_backend(name, klass)
@@ -36,7 +46,7 @@ module Fluent
 
     class Backend
       UpdateMode = MetricSenseOutput::UpdateMode
-      include Configurable
+      include Fluent::Configurable
 
       attr_accessor :log
 
@@ -69,11 +79,8 @@ module Fluent
 
     config_param :aggregate_interval, :time, :default => 60
 
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
-
     def configure(conf)
+      compat_parameters_convert(conf, :buffer)
       super
 
       if @remove_tag_prefix
@@ -111,81 +118,62 @@ module Fluent
     end
 
     def shutdown
-      super
       @backend.shutdown
+      super
     end
 
-    def format_stream(tag, es)
-      # modify tag
+    def format(tag, time, record)
       tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
-      tag = "#{add_tag_prefix}.#{tag}" if @add_tag_prefix
-
-      out = ''
-      es.each do |time,record|
-        # dup record to modify
-        record = record.dup
+      tag = "#{@add_tag_prefix}.#{tag}" if @add_tag_prefix
 
-        # get value
-        value = record.delete(@value_key)
+      record = record.dup
 
-        # ignore record if value is missing
-        next if value.nil?
+      value = record.delete(@value_key)
+      return nil if value.nil?
 
-        # ignore record if value is invalid
-        begin
-          fv = value.to_f
-        rescue
-          next
-        end
-        next if fv.nan? || fv.infinite?
-
-        # use integer if value.to_f == value.to_f.to_i
-        iv = fv.to_i
-        if iv.to_f == fv
-          value = iv
-        else
-          value = fv
-        end
-
-        # get update_mode key
-        update_mode = record.delete(@update_mode_key)
-        case update_mode
-        when "max"
-          update_mode = UpdateMode::MAX
-        when "average"
-          update_mode = UpdateMode::AVERAGE
-        when "count"
-          update_mode = UpdateMode::COUNT
-        else
-          # default is add
-          update_mode = UpdateMode::ADD
-        end
+      begin
+        fv = value.to_f
+      rescue
+        return nil
+      end
+      return nil if fv.nan? || fv.infinite?
+
+      iv = fv.to_i
+      value = iv.to_f == fv ? iv : fv
+
+      update_mode = record.delete(@update_mode_key)
+      case update_mode
+      when "max"
+        update_mode = UpdateMode::MAX
+      when "average"
+        update_mode = UpdateMode::AVERAGE
+      when "count"
+        update_mode = UpdateMode::COUNT
+      else
+        update_mode = UpdateMode::ADD
+      end
 
-        # get segments
-        if @no_segment_keys
+      if @no_segment_keys
+        segments = {}
+      else
+        if @only_segment_keys
           segments = {}
+          @only_segment_keys.each {|key|
+            if v = record[key]
+              segments[key] = v
+            end
+          }
         else
-          if @only_segment_keys
-            segments = {}
-            @only_segment_keys.each {|key|
-              if v = record[key]
-                segments[key] = v
-              end
-            }
-          else
-            segments = record
-          end
-          if @exclude_segment_keys
-            @exclude_segment_keys.each {|key|
-              segments.delete(key)
-            }
-          end
+          segments = record
+        end
+        if @exclude_segment_keys
+          @exclude_segment_keys.each {|key|
+            segments.delete(key)
+          }
         end
-
-        [tag, time, value, segments, update_mode].to_msgpack(out)
       end
 
-      out
+      [tag, time.to_i, value, segments, update_mode].to_msgpack
     end
 
     class AddUpdater
@@ -262,30 +250,34 @@ module Fluent
 
       # select sum(value) from chunk group by tag, time/60, seg_val, seg_key
       chunk.msgpack_each {|tag,time,value,segments,update_mode|
-        time = time / @aggregate_interval * @aggregate_interval
-
-        case update_mode
-        when UpdateMode::ADD
-          updater = AddUpdater
-        when UpdateMode::MAX
-          updater = MaxUpdater
-        when UpdateMode::AVERAGE # AVERAGE uses MaxUpdater and calculate average on server-side aggregation
-          updater = AverageUpdater
-        when UpdateMode::COUNT
-          updater = CountUpdater
-        else  # default is AddUpdater
-          updater = AddUpdater
-        end
-
-        # simple values
-        ak = AggregationKey.new(tag, time, nil, nil)
-        (simple_counters[ak] ||= updater.new).add(value)
+        begin
+          time = time / @aggregate_interval * @aggregate_interval
+
+          case update_mode
+          when UpdateMode::ADD
+            updater = AddUpdater
+          when UpdateMode::MAX
+            updater = MaxUpdater
+          when UpdateMode::AVERAGE # AVERAGE uses MaxUpdater and calculate average on server-side aggregation
+            updater = AverageUpdater
+          when UpdateMode::COUNT
+            updater = CountUpdater
+          else  # default is AddUpdater
+            updater = AddUpdater
+          end
 
-        # segmented values
-        segments.each_pair {|seg_key,seg_val|
-          ak = AggregationKey.new(tag, time, seg_val, seg_key)
-          (segmented_counters[ak] ||= updater.new).add(value)
-        }
+          # simple values
+          ak = AggregationKey.new(tag, time, nil, nil)
+          (simple_counters[ak] ||= updater.new).add(value)
+
+          # segmented values
+          segments.each_pair {|seg_key,seg_val|
+            ak = AggregationKey.new(tag, time, seg_val, seg_key)
+            (segmented_counters[ak] ||= updater.new).add(value)
+          }
+        rescue StandardError => e
+          log.warn("ignoring broken chunk: #{e.inspect} - " + [tag, time, value, segments, update_mode].inspect)
+        end
       }
 
       counters = segmented_counters

data/test/plugin/test_out_metricsense.rb

@@ -0,0 +1,118 @@
+require 'test-unit'
+require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+require 'fluent/plugin/out_metricsense'
+
+class MetricsenseOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  class TestBackend < Fluent::MetricSenseOutput::Backend
+    Fluent::MetricSenseOutput.register_backend('test', self)
+
+    @@data = []
+
+    def self.data
+      @@data
+    end
+
+    def self.clear
+      @@data = []
+    end
+
+    def write(data)
+      @@data << data
+    end
+  end
+
+  CONFIG = %Q[
+    backend test
+  ]
+
+  def setup
+    Fluent::Test.setup
+    TestBackend.clear
+  end
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::MetricSenseOutput).configure(conf)
+  end
+
+  def test_emit
+    now = Time.now.to_i
+    d = create_driver
+    d.run(default_tag: 'test') do
+      d.feed(now, {'value' => 1, 'user_id' => 23456, 'path' => '/auth/login'})
+    end
+
+    t = now / 60 * 60
+    assert_true TestBackend.data.length > 0
+    TestBackend.data.each do |written|
+      assert_equal ['test', t, 1, 'user_id', 23456, 0], written.shift
+      assert_equal ['test', t, 1, 'path', "/auth/login", 0], written.shift
+      assert_equal ['test', t, 1, nil, nil, 0], written.shift
+    end
+  end
+
+  # server restart can cause broken buffer and broken chunk.
+  # out_metricsense should ignore the chunk with warning log.
+  # Note: v1 driver validates inputs, so we hand-roll a corrupt chunk
+  # to exercise write's error handling directly.
+  def test_skip_broken_chunk
+    d = create_driver
+
+    corrupt_chunk = Object.new
+    def corrupt_chunk.msgpack_each
+      yield ['test', 'not_a_number', 1, {}, 0]
+    end
+
+    assert_nothing_raised do
+      d.instance.write(corrupt_chunk)
+    end
+    assert_equal [[]], TestBackend.data
+  end
+
+  def test_format_skips_nil_value
+    now = Time.now.to_i
+    d = create_driver
+    d.run(default_tag: 'test') do
+      d.feed(now, {'no_value_key' => 'x'})
+      d.feed(now, {'value' => 1})
+    end
+
+    assert_nil d.formatted.first
+    assert_equal 1, d.formatted.compact.length
+    assert_equal 1, TestBackend.data.flatten(1).length
+  end
+
+  def test_format_skips_nan_value
+    now = Time.now.to_i
+    d = create_driver
+    d.run(default_tag: 'test') do
+      d.feed(now, {'value' => Float::NAN})
+      d.feed(now, {'value' => 1})
+    end
+
+    assert_nil d.formatted.first
+    assert_equal 1, d.formatted.compact.length
+    assert_equal 1, TestBackend.data.flatten(1).length
+  end
+
+  def test_format_skips_infinite_value
+    now = Time.now.to_i
+    d = create_driver
+    d.run(default_tag: 'test') do
+      d.feed(now, {'value' => Float::INFINITY})
+      d.feed(now, {'value' => 1})
+    end
+
+    assert_nil d.formatted.first
+    assert_equal 1, d.formatted.compact.length
+    assert_equal 1, TestBackend.data.flatten(1).length
+  end
+
+  def test_reloadable
+    d = create_driver
+    assert_true d.instance.reloadable_plugin?
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-metricsense
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-06 00:00:00.000000000 Z
+date: 2026-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.6
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.6
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: dogapi
   requirement: !ruby/object:Gem::Requirement
@@ -66,20 +72,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.1
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.10.0
 description: MetricSense - application metrics aggregation plugin for Fluentd
 email: sf@treasure-data.com
 executables: []
@@ -91,6 +111,7 @@ files:
 - LICENSE
 - README.rdoc
 - Rakefile
+- SECURITY.md
 - VERSION
 - fluent-plugin-metricsense.gemspec
 - lib/fluent/plugin/backends/datadog_backend.rb
@@ -98,6 +119,7 @@ files:
 - lib/fluent/plugin/backends/rdb_tsdb_backend.rb
 - lib/fluent/plugin/backends/stdout_backend.rb
 - lib/fluent/plugin/out_metricsense.rb
+- test/plugin/test_out_metricsense.rb
 homepage: https://github.com/treasure-data/fluent-plugin-metricsense
 licenses:
 - Apache-2.0
@@ -117,9 +139,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: MetricSense - application metrics aggregation plugin for Fluentd
-test_files: []
+test_files:
+- test/plugin/test_out_metricsense.rb