fluent-plugin-cmetrics 0.1.0.rc4 → 0.1.0.rc8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0cb7fd9484f58d44c52df4a1420bb4ecfc3cd612e47e223bd7dd74871e477c3
-  data.tar.gz: 0cdeabbb2069ccac75b15337f9f5141909944d5bc70ad4e6a46cecc4b6d82904
+  metadata.gz: e207c02b8b1d7751c344573ec09c0d1ccf56c83d99776b2dade2e3b1366cb3d7
+  data.tar.gz: 448a06563a66ff12155a5059598c6fb99334f178f4564c0220c7baaecbc19eed
 SHA512:
-  metadata.gz: 9bd6c7e03deeceae65fc856e89f8340e2c207e27aa77cf83805399d6475db7662aea8fb421c2c1314dc3cc35d2ec9f20f8d0ab2dba7c53efee6d3e06a26fd947
-  data.tar.gz: 7349f9dd27645fbe3f8fd024bdd00e8db31c5ac728642a75bc8570157098a76827b7308e4b9526b4d3d34fbd54b81f73eb9c0f27fae3b55b65e8be1201cb359a
+  metadata.gz: a7219f2904e99a3b8f48f6fbaaa89d4261169563919be1d02552322b78b2756e56218e1982ef36bb93932813659e7c20ef40724f2768d6c2568f0b2fab24fc7f
+  data.tar.gz: fee515f5d3ae1cf09ce5af4fe4e45a6a471255d9db3ab4e9226c53cf74ea5146e764956fba8e78d16ddb65fee5e6a8b48530213d9c1d0e075cd54864d5fc4ec5

data/README.md

@@ -32,22 +32,87 @@ $ bundle
 
 ## Fluent::Plugin::CMetricsParserFilter
 
-### cmetric_metric_key (string) (optional)
+### cmetrics_metric_key (string) (optional)
 
 cmetrics metric key
 
 Default value: `cmetrics`.
 
-### cmetric_labels_key (string) (optional)
+### cmetrics_labels_key (string) (optional)
 
 cmetrics labels key
 
 Default value: `labels`.
 
-### format_name_key_for_splunk_metric (bool) (optional)
+### format_to_splunk_metric (bool) (optional)
 
-format name key for Splunk metrics
+format to Splunk metrics
 
+### dimensions_key (string) (optional)
+
+dimensions key
+
+
+### \<fields\> section (optional) (single)
+
+This secsion is used for adding extra fields into cmetrics msgpack payload parsed records.
+
+For example, the following configuration should add hostname records into parsed records:
+
+```aconf
+<filter super.awesome.tag.**>
+  @type cmetrics_parser
+  format_to_splunk_metric true
+  dimensions_key dims
+  <fields>
+    hostname
+  </fields>
+</filter>
+```
+
+On later data pipeline, `hostname` key can be used as some additional work.
+
+## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
+
+### cmetrics_name_key (string) (optional)
+
+cmetrics metrics name key
+
+Default value: `name`.
+
+### cmetrics_value_key (string) (optional)
+
+cmetrics metrics value key
+
+Default value: `value`.
+
+### cmetrics_dims_key (string) (optional)
+
+cmetrics metrics dimensions key
+
+Default value: `dims`.
+
+### host_key (string) (optional)
+
+Specify host key
+
+Default value: `host`.
+
+### index (string) (optional)
+
+Specify splunk index name
+
+### source (string) (optional)
+
+Specify splunk source name
+
+### sourcetype (string) (optional)
+
+Specify splunk sourcetype name
+
+### \<fields\> section (optional) (single)
+
+This secsion is used for adding extra fields as dimensions into cmetrics msgpack Splunk metric payload.
 
 ## Copyright
 

data/fluent-plugin-cmetrics.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-cmetrics"
-  spec.version = "0.1.0.rc4"
+  spec.version = "0.1.0.rc8"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["cosmo0920.oucc@gmail.com"]
 

data/lib/fluent/plugin/filter_cmetrics_parser.rb

@@ -30,43 +30,66 @@ module Fluent
       config_param :cmetrics_metric_key, :string, default: "cmetrics"
       desc "cmetrics labels key"
       config_param :cmetrics_labels_key, :string, default: "labels"
-      desc "format name key for Splunk metrics"
-      config_param :format_name_key_for_splunk_metric, :bool, default: false
+      desc "format to Splunk metrics"
+      config_param :format_to_splunk_metric, :bool, default: false
+      desc "dimensions key"
+      config_param :dimensions_key, :string, default: nil
+      desc "Add additional records for Splunk dimensions"
+      config_section :fields, init: false, multi: false,required: false do
+        # Nothing here. For later purpose.
+      end
 
       def configure(conf)
         super
         @serde = ::CMetrics::Serde.new
         @record_accessor = record_accessor_create(@cmetrics_metric_key)
         @labels_accessor = record_accessor_create(@cmetrics_labels_key)
+        @fields_accessors = {}
+        conf.elements(name: "fields").each do |e|
+          e.each_pair{|k, _v|
+            e.has_key?(k) # Suppress unused warnings.
+            @fields_accessors[k] = record_accessor_create(k)
+          }
+        end
       end
 
-      def format_record_key_to_splunk_style(inner)
+      def format_to_splunk_style_with_dims(inner)
         subsystem = inner.delete("subsystem")
-        labels_str = if labels = @labels_accessor.call(inner)
-                       labels_str = labels.map {|k,v|
-                         if k == "cpu"
-                           "id.#{v}"
-                         else
-                           "#{k}.#{v}"
-                         end
-                       }.join(".")
-                     end
+        # labels will be treated as dimensions.
+        dimensions = Hash.new(0)
+        if labels = @labels_accessor.call(inner)
+          labels.map {|k,v|
+            dimensions[k] = v
+          }
+        end
         name = inner.delete("name")
-        [subsystem, labels_str, name].compact.reject{|e| e.empty?}.join(".")
+        return [subsystem, name].compact.reject{|e| e.empty?}.join("."), dimensions
       end
 
       def filter_stream(tag, es)
         new_es = Fluent::MultiEventStream.new
         es.each do |time, record|
           data = @record_accessor.call(record)
+          extra_fields = {}
+          @fields_accessors.each do |key, accessor|
+            extra_fields[key] = accessor.call(record)
+          end
           @serde.feed_each(data) do |cmetrics|
             metrics = cmetrics.metrics
             metrics.each do |metric|
               next if metric.empty?
 
               metric.each do |inner|
-                if @format_name_key_for_splunk_metric
-                  inner["name"] = format_record_key_to_splunk_style(inner)
+                if @format_to_splunk_metric
+                  inner["name"], dims = format_to_splunk_style_with_dims(inner)
+                  if @dimensions_key
+                    inner[@dimensions_key] = dims
+                  else
+                    inner.merge!(dims)
+                  end
+                end
+                if @fields_accessors
+                  inner.merge!(extra_fields)
                 end
                 time = Time.at(inner.delete("timestamp"))
                 new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)

data/lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb

@@ -34,6 +34,8 @@ module Fluent
       config_param :cmetrics_name_key, :string, default: "name"
       desc "cmetrics metrics value key"
       config_param :cmetrics_value_key, :string, default: "value"
+      desc "cmetrics metrics dimensions key"
+      config_param :cmetrics_dims_key, :string, default: "dims"
       desc "Specify host key"
       config_param :host_key, :string, default: "host"
       desc "Specify splunk index name"
@@ -42,6 +44,9 @@ module Fluent
       config_param :source, :string, default: nil
       desc "Specify splunk sourcetype name"
       config_param :sourcetype, :string, default: nil
+      config_section :fields, init: false, multi: false,required: false do
+        # Nothing here. For later purpose.
+      end
 
       def initialize
         super
@@ -53,7 +58,15 @@ module Fluent
 
         @cmetrics_name_accessor = record_accessor_create(@cmetrics_name_key)
         @cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
+        @cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
         @host_key_accessor = record_accessor_create(@host_key)
+        @fields_accessors = {}
+        conf.elements(name: "fields").each do |e|
+          e.each_pair{|k, _v|
+            e.has_key?(k) # Suppress unused warnings.
+            @fields_accessors[k] = record_accessor_create(k)
+          }
+        end
       end
 
       def format(tag, time, record)
@@ -62,6 +75,11 @@ module Fluent
                else
                  @default_host
                end
+        extra_fields = {}
+
+        @fields_accessors.each do |key, accessor|
+          extra_fields[key] = accessor.call(record)
+        end
         payload = {
           host: host,
           # From the API reference
@@ -72,11 +90,16 @@ module Fluent
         payload[:index] = @index if @index
         payload[:source] = @source if @source
         payload[:sourcetype] = @sourcetype if @sourcetype
-
-        metric_fields = {
+        fields = {
           "metric_name:#{@cmetrics_name_accessor.call(record)}" => @cmetrics_value_accessor.call(record)
         }
-        payload.merge!(metric_fields)
+        if dims = @cmetrics_dims_accessor.call(record)
+          fields.merge!(dims)
+        end
+        if @fields_accessors
+          fields.merge!(extra_fields)
+        end
+        payload.merge!(fields)
         Yajl.dump(payload)
       end
     end

data/test/plugin/test_filter_cmetrics_parser.rb

@@ -1,21 +1,27 @@
 require "helper"
 require "fluent/plugin/filter_cmetrics_parser.rb"
+require 'socket'
 
 class CmetricsParserTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
   end
 
-  data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.id.10.mode.system.seconds_total"],
-       "filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum"],
-       "disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.device.nvme0n1.io_now"],
-       "network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.device.eth0.transmit_bytes_total"],
-      "none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5"])
-  test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key)|
+  data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.seconds_total", {"cpu" => "10", "mode" => "system"}],
+       "filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum", {}],
+       "disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.io_now", {"device" => "nvme0n1"}],
+       "network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.transmit_bytes_total", {"device" => "eth0"}],
+      "none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5", {}])
+  test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key, expected_dims)|
     json = Yajl.load(json_str)
-    d = create_driver(%[format_name_key_for_splunk_metric true])
-    assert_true d.instance.format_name_key_for_splunk_metric
-    assert_equal expected_format_key, d.instance.format_record_key_to_splunk_style(json)
+    d = create_driver(%[
+      format_to_splunk_metric true
+      dimensions_key dims
+    ])
+    assert_true d.instance.format_to_splunk_metric
+    formatted_key, dims = d.instance.format_to_splunk_style_with_dims(json)
+    assert_equal expected_format_key, formatted_key
+    assert_equal expected_dims, dims
   end
 
   sub_test_case "Actual filtering" do
@@ -24,13 +30,57 @@ class CmetricsParserTest < Test::Unit::TestCase
       @binary = File.read(@binary_path)
     end
 
-    test "#filter_stream" do
-      d = create_driver(%[format_name_key_for_splunk_metric true])
+    data("with dimensions" => "dims",
+         "without dimensions" => nil)
+    test "#filter_stream" do |data|
+      use_dimensions = data
+      d = if use_dimensions
+            create_driver(%[
+              format_to_splunk_metric true
+              dimensions_key dims
+            ])
+          else
+            create_driver(%[
+              format_to_splunk_metric true
+            ])
+          end
       time = event_time("2012-01-02 13:14:15")
       record = {"cmetrics" => @binary}
       d.run(default_tag: 'test') do
         d.feed(time, record)
       end
+      d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
+      assert do
+        d.filtered.size > 0
+      end
+    end
+
+    data("with dimensions" => "dims",
+         "without dimensions" => nil)
+    test "#filter_stream with host_key" do |data|
+      use_dimensions = data
+      d = if use_dimensions
+            create_driver(Fluent::Config::Element.new('ROOT', '', {
+                                                        "format_to_splunk_metric" => true,
+                                                        "dimensions_key" => "dims",
+                                                      }, [
+                                                        Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
+                                                      ]))
+          else
+            create_driver(Fluent::Config::Element.new('ROOT', '', {
+                                                        "format_to_splunk_metric" => true,
+                                                      }, [
+                                                        Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
+                                                      ]))
+          end
+      time = event_time("2012-01-02 13:14:15")
+      record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
+      d.run(default_tag: 'test') do
+        d.feed(time, record)
+      end
+      d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
       assert do
         d.filtered.size > 0
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cmetrics
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc4
+  version: 0.1.0.rc8
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-22 00:00:00.000000000 Z
+date: 2021-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler