fluent-plugin-watch-objectspace 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50ef80ccfe4236df352654ccbf415698e15bcf98927fc6bebba8ddbffd3d76bb
-  data.tar.gz: 25efcc62552470384eebf776edcd6dc89e45d7550319dd2bd141b4f4608e6d7c
+  metadata.gz: f79b912a0f6dc31b0b0a11ebec4780ecdb56abd848842b1250440afe15995ab1
+  data.tar.gz: 343fceb8323cc98295d3c0fe1b52a5b0d8eff59f061bbd8f8191f876af2af3d0
 SHA512:
-  metadata.gz: '0935e7413eefa8bc4302b0d151c403fa24131960f3aa9dddc87c2c815dab3321328b007a51d5b829a9019c3b5e2450d4fcb48d8aff23f5cbc29e226bc783a8bc'
-  data.tar.gz: 73f7efa2832cce3b36f024b9c1feb4ac729a20aa63426b583b5012ff784391af7150c26a732471ff14aee03342d01647deb7101b35c95a06f5f016dc37c2001d
+  metadata.gz: 4d6b10a8abcbbe443c81f1f59bcc44610d62d451dedfeaeab0c458d76aee9459854c1197e6ad0f3a5b30fbc34d357b71815e40251189dba6921cc562dcdd397b
+  data.tar.gz: 606d74ca77ac3aa93ce6113d0c318e7777caed306feda3c80eba93ccce68dce0ef3b262672698783adef649c0a33cf7607b8e4891bf6348ada076fdb4670d3dd

data/README.md

@@ -58,10 +58,10 @@ If memory usage is over 1.3 times, it raise an exception.
 
 ## FAQ
 
-# What is the different between fluent-plugin-watch-objectspace and fluent-plugin-watch-process
+### What is the difference between fluent-plugin-watch-objectspace and fluent-plugin-watch-process?
 
 fluent-plugin-watch-process is useful cron/batch process monitoring, In contrast to it, fluent-plugin-watch-objectspace is
-focused on used plugin's resource (memory) usage especially object and memory.
+focused on used plugin's resource usage especially object and memory.
 
 ## Copyright
 

data/fluent-plugin-watch-objectspace.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-watch-objectspace"
-  spec.version = "0.1.0"
+  spec.version = "0.2.0"
   spec.authors = ["Kentaro Hayashi"]
   spec.email   = ["kenhys@gmail.com"]
 

data/lib/fluent/plugin/in_watch_objectspace.rb

@@ -14,6 +14,8 @@
 # limitations under the License.
 
 require "fluent/plugin/input"
+require "fluent/config/error"
+require "objspace"
 
 module Fluent
   module Plugin
@@ -34,14 +36,25 @@ module Fluent
       config_param :watch_delay, :time, default: 60
       desc "Collect GC::Profiler.raw_data"
       config_param :gc_raw_data, :bool, default: false
-      desc "Threshold rate which regards increased RES as memory leaks"
-      config_param :res_incremental_threshold_rate, :float, default: 1.3
-     
+      desc "Specify included fields of top command"
+      config_param :top_fields, :array, default: ["VIRT", "RES", "SHR", "%CPU", "%MEM", "TIME+"]
+
+      config_section :threshold, required: false, multi: false do
+        desc "Threshold rate which regards increased memsize as memory leaks"
+        config_param :memsize_of_all, :float, default: 1.3
+        desc "Threshold rate which regards increased RES as memory leaks"
+        config_param :res_of_top, :float, default: nil
+      end
+
       def configure(conf)
         super(conf)
         if @modules
           @modules.each do |mod|
-            require mod
+            begin
+              require mod
+            rescue LoadError
+              raise Fluent::ConfigError.new("BUG: module <#{mod}> can't be loaded")
+            end
           end
         end
         @warmup_time = Time.now + @watch_delay
@@ -59,6 +72,7 @@ module Fluent
         fields = content.split("\n")[-2].split
         values = content.split("\n").last.split
         fields.each_with_index do |field, index|
+          next unless @top_fields.include?(field)
           case field
           when "USER", "S", "TIME+", "COMMAND"
             record[field.downcase] = values[index]
@@ -78,7 +92,8 @@ module Fluent
         record = {
           "pid" => pid,
           "count" => {},
-          "memory_leaks" => false
+          "memory_leaks" => false,
+          "memsize_of_all" => ObjectSpace.memsize_of_all
         }
 
         begin
@@ -88,23 +103,20 @@ module Fluent
           record.merge!(parse_top_result(content))
 
           if @gc_raw_data
-            record["raw_data"] = GC::Profiler.raw_data
+            record["gc_raw_data"] = GC::Profiler.raw_data
           end
-          @watch_class.each do |klass|
-            record["count"]["#{klass.downcase}"] = ObjectSpace.each_object(Object.const_get(klass)) { |x| x }
+          if @watch_class
+            @watch_class.each do |klass|
+              record["count"]["#{klass.downcase}"] = ObjectSpace.each_object(Object.const_get(klass)) { |x| x }
+            end
           end
 
           if @source.empty?
+            record["memsize_of_all"] = ObjectSpace.memsize_of_all
             @source = record
           end
 
-          if @source["res"] * @res_incremental_threshold_rate < record["res"]
-            record["memory_leaks"] = true
-            message = sprintf("Memory leak is detected, threshold rate <%f>: %f > %f * %f",
-                              @res_incremental_threshold_rate, record["res"],
-                              @source["res"], @res_incremental_threshold_rate)
-            raise message
-          end
+          check_threshold(record)
           es = OneEventStream.new(Fluent::EventTime.now, record)
           router.emit_stream(@tag, es)
         rescue => e
@@ -112,6 +124,29 @@ module Fluent
         end
       end
 
+      def check_threshold(record)
+        return unless @threshold
+
+        if @threshold.res_of_top
+          if @source["res"] * @threshold.res_of_top < record["res"]
+            record["memory_leaks"] = true
+            message = sprintf("Memory leak is detected, threshold res_of_top rate <%f>: %f > %f * %f",
+                              @threshold.res_of_top, record["res"],
+                              @source["res"], @threshold.res_of_top)
+            raise message
+          end
+        end
+        if @threshold.memsize_of_all
+          if @source["memsize_of_all"] * @threshold.memsize_of_all < record["memsize_of_all"]
+            record["memory_leaks"] = true
+            message = sprintf("Memory leak is detected, threshold of memsize_of_all rate <%f>: %f > %f * %f",
+                              @threshold.memsize_of_all, record["memsize_of_all"],
+                              @source["memsize_of_all"], @threshold.memsize_of_all)
+            raise message
+          end
+        end
+      end
+
       def shutdown
       end
     end

data/test/plugin/test_in_watch_objectspace.rb

@@ -14,28 +14,197 @@ class WatchObjectspaceInputTest < Test::Unit::TestCase
     Fluent::Test::Driver::Input.new(Fluent::Plugin::WatchObjectspaceInput).configure(conf)
   end
 
+  def create_config(params={})
+    config_element("ROOT", "", params)
+  end
+
+  def default_params(data={})
+    {
+      "watch_delay" => 0,
+      "watch_interval" => 1
+    }.merge(data)
+  end
+
   sub_test_case "configure" do
     def test_default_configuration
       d = create_driver
       assert_equal([
+                     nil,
+                     60,
+                     60,
+                     "watch_objectspace",
+                     nil,
+                     false
+                   ],
+                   [
+                     d.instance.watch_class,
+                     d.instance.watch_interval,
+                     d.instance.watch_delay,
+                     d.instance.tag,
+                     d.instance.modules,
+                     d.instance.gc_raw_data
+                   ])
+    end
+
+    def test_customize
+      config = config_element("ROOT", "", {
+                                "watch_delay" => 1,
+                                "watch_interval" => 2,
+                                "watch_class" => ["String"],
+                                "tag" => "customized",
+                                "modules" => ["objspace"],
+                                "gc_raw_data" => true,
+                                "res_incremental_threshold_rate" => 1.1,
+                                "memsize_of_all_incremental_threshold_rate" => 1.5
+                              })
+      d = create_driver(config)
+      assert_equal([
+                     ["String"],
+                     2.0,
+                     1.0,
+                     "customized",
+                     ["objspace"],
+                     true
+                   ],
+                   [
                      d.instance.watch_class,
                      d.instance.watch_interval,
                      d.instance.watch_delay,
                      d.instance.tag,
                      d.instance.modules,
                      d.instance.gc_raw_data,
-                     d.instance.res_incremental_threshold_rate
+                   ])
+    end
+
+    def test_watch_class
+      config = create_config(default_params({"watch_class" => ["String"]}))
+      d = create_driver(config)
+      d.run(expect_records: 1, timeout: 1)
+      assert_equal([
+                     1,
+                     ["string"],
+                     true
                    ],
                    [
-                     nil,
-                     60,
-                     60,
-                     "watch_objectspace",
-                     nil,
-                     false,
-                     1.3
+                     d.events.size,
+                     d.events.first.last["count"].keys,
+                     d.events.first.last["count"]["string"] > 0
                    ])
     end
+
+    def test_watch_interval
+      config = create_config(default_params({"watch_interval" => 5}))
+      d = create_driver(config)
+      d.run(expect_records: 1, timeout: 5)
+      assert_equal(1, d.events.size)
+    end
+
+    sub_test_case "watch_delay" do
+      data(
+        before_interval: [1, 0, 5],
+        after_interval: [0, 10, 5]
+      )
+      test "watch delay" do |(count, delay, interval)|
+        config = create_config(default_params({"watch_delay" => delay, "watch_interval" => interval}))
+        d = create_driver(config)
+        d.run(expect_records: 1, timeout: interval)
+        assert_equal(count, d.events.size)
+      end
+    end
+
+    sub_test_case "tag" do
+      data(
+        with_tag: ["changed", "changed"],
+        default: ["watch_objectspace", nil]
+      )
+      test "tag" do |(tag, specified)|
+        config = if specified
+                   create_config(default_params({"tag" => specified}))
+                 else
+                   create_config(default_params)
+                 end
+        d = create_driver(config)
+        d.run(expect_records: 1, timeout: 5)
+        assert_equal([tag], d.events.collect { |event| event.first })
+      end
+    end
+
+    sub_test_case "modules" do
+      def test_invalid_module
+        config = create_config(default_params({"modules" => "404",
+                                               "watch_class" => "404"}))
+        assert_raise do
+          create_driver(config)
+        end
+      end
+
+      def test_valid_module
+        config = create_config(default_params({"modules" => "fluent/plugin/in_watch_objectspace",
+                                               "watch_class" => "Fluent::Plugin::WatchObjectspaceInput"}))
+        d = create_driver(config)
+        d.run(expect_records: 1, timeout: 1)
+        assert_equal([
+                       ["fluent::plugin::watchobjectspaceinput"],
+                       true
+                     ],
+                     [
+                       d.events.collect { |event| event.last["count"].keys }.flatten,
+                       d.events.all? { |event| event.last["count"]["fluent::plugin::watchobjectspaceinput"] > 0 }
+                     ])
+      end
+    end
+
+    def gc_raw_data_keys(data)
+      if data.empty?
+        data
+      else
+        data.collect do |raw_data|
+          raw_data.keys
+        end
+      end
+    end
+
+    sub_test_case "gc_raw_data" do
+      data(
+        without_gc: [false, [[]]],
+        with_gc: [true, [[%i(GC_FLAGS GC_TIME GC_INVOKE_TIME HEAP_USE_SIZE HEAP_TOTAL_SIZE HEAP_TOTAL_OBJECTS GC_IS_MARKED)]]]
+      )
+      test "gc" do |(gc, keys)|
+        config = create_config(default_params({"gc_raw_data" => "true"}))
+        d = create_driver(config)
+        GC::Profiler.clear
+        GC.start if gc
+        d.run(expect_records: 1, timeout: 1)
+        assert_equal([
+                       1,
+                       keys
+                     ],
+                     [
+                       d.events.size,
+                       d.events.collect { |event| gc_raw_data_keys(event.last["gc_raw_data"])},
+                     ])
+      end
+    end
+
+    sub_test_case "threshold" do
+      def test_memsize_of_all
+        config = config_element("ROOT", "", {
+                                  "watch_delay" => 0,
+                                  "watch_interval" => 2,
+                                }, [config_element("threshold", "", {})])
+        d = create_driver(config)
+        assert_equal(1.3, d.instance.threshold.memsize_of_all)
+      end
+
+      def test_res_of_top
+        config = config_element("ROOT", "", {
+                                  "watch_delay" => 0,
+                                  "watch_interval" => 2,
+                                }, [config_element("threshold", "", {"res_of_top" => 1.5})])
+        d = create_driver(config)
+        assert_equal(1.5, d.instance.threshold.res_of_top)
+      end
+    end
   end
 
   sub_test_case "parser" do
@@ -50,7 +219,7 @@ class WatchObjectspaceInputTest < Test::Unit::TestCase
       assert_equal([1,
                     "watch_objectspace",
                     Fluent::EventTime,
-                    ["pid", "count", "memory_leaks", "user", "pr", "ni", "virt", "res", "shr", "s", "%cpu", "%mem", "time+", "command"]
+                    ["pid", "count", "memory_leaks", "memsize_of_all", "virt", "res", "shr", "%cpu", "%mem", "time+"]
                    ],
                    [d.events.size,
                     event[0],

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-watch-objectspace
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Kentaro Hayashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-27 00:00:00.000000000 Z
+date: 2021-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler