fluent-plugin-cmetrics 0.1.0.rc3 → 0.1.0.rc7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c618e0924f2b3abe13bb52238e97e86bc4fbbdc15ca67abd257228cd7c051538
-  data.tar.gz: c78308c6d609b74695e35dbe315baffaca9f129e7de1e83102d16f4a1a27f5e2
+  metadata.gz: f2da1d6e9e9b821b5992d1e6989d034d04f074bc10a11ecd35d2842207a77754
+  data.tar.gz: c71b5a5c8ad4a33328b4c542d00a2a562639fa9f9b646db3bd9c66cf922c2b53
 SHA512:
-  metadata.gz: e564d3e9d388eb9668df6c47d6420d5402f42617d7b104d3aa7ca6de2c92d74fba549809fd6d3908eaa782a3dc40b7c6934a920d8170f6e03327a66c94c7408b
-  data.tar.gz: ecf3f9708f339b80b28622be0da128d23d255868f1ce4fb98abaaf7ffa643caa366b8723013390ec8fa2f28c2a22a3d333ba2149ffe9af317df89e3a1d38062a
+  metadata.gz: 5cd27f38bb9ea19e325589293f29cf13b07bf3158adb0d21febc502d6f5c2e76ed8841ebea47436478890c79fa9ac546918abd91bce17f48ef2970a71695bc38
+  data.tar.gz: 6661de3897ecac7c2a07a5765dbf16f9c6876e6ab2d6ae80d789793473c67ca91ac3fe4a8091335f99dcc517c4343253e3369c5cdd2c977d3d4141a85b82fd19

data/README.md

@@ -32,22 +32,83 @@ $ bundle
 
 ## Fluent::Plugin::CMetricsParserFilter
 
-### cmetric_metric_key (string) (optional)
+### cmetrics_metric_key (string) (optional)
 
 cmetrics metric key
 
 Default value: `cmetrics`.
 
-### cmetric_labels_key (string) (optional)
+### cmetrics_labels_key (string) (optional)
 
 cmetrics labels key
 
 Default value: `labels`.
 
-### format_name_key_for_splunk_metric (bool) (optional)
+### format_to_splunk_metric (bool) (optional)
 
-format name key for Splunk metrics
+format to Splunk metrics
 
+### dimensions_key (string) (optional)
+
+dimensions key
+
+
+### \<fields\> section (optional) (single)
+
+This secsion is used for adding extra fields into cmetrics msgpack payload parsed records.
+
+For example, the following configuration should add hostname records into parsed records:
+
+```aconf
+<filter super.awesome.tag.**>
+  @type cmetrics_parser
+  format_to_splunk_metric true
+  dimensions_key dims
+  <fields>
+    hostname
+  </fields>
+</filter>
+```
+
+On later data pipeline, `hostname` key can be used as some additional work.
+
+## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
+
+### cmetrics_name_key (string) (optional)
+
+cmetrics metrics name key
+
+Default value: `name`.
+
+### cmetrics_value_key (string) (optional)
+
+cmetrics metrics value key
+
+Default value: `value`.
+
+### cmetrics_dims_key (string) (optional)
+
+cmetrics metrics dimensions key
+
+Default value: `dims`.
+
+### host_key (string) (optional)
+
+Specify host key
+
+Default value: `host`.
+
+### index (string) (optional)
+
+Specify splunk index name
+
+### source (string) (optional)
+
+Specify splunk source name
+
+### sourcetype (string) (optional)
+
+Specify splunk sourcetype name
 
 ## Copyright
 

data/fluent-plugin-cmetrics.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-cmetrics"
-  spec.version = "0.1.0.rc3"
+  spec.version = "0.1.0.rc7"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["cosmo0920.oucc@gmail.com"]
 

data/lib/fluent/plugin/filter_cmetrics_parser.rb

@@ -30,43 +30,66 @@ module Fluent
       config_param :cmetrics_metric_key, :string, default: "cmetrics"
       desc "cmetrics labels key"
       config_param :cmetrics_labels_key, :string, default: "labels"
-      desc "format name key for Splunk metrics"
-      config_param :format_name_key_for_splunk_metric, :bool, default: false
+      desc "format to Splunk metrics"
+      config_param :format_to_splunk_metric, :bool, default: false
+      desc "dimensions key"
+      config_param :dimensions_key, :string, default: nil
+      desc "Add additional records for Splunk dimensions"
+      config_section :fields, init: false, multi: false,required: false do
+        # Nothing here. For later purpose.
+      end
 
       def configure(conf)
         super
         @serde = ::CMetrics::Serde.new
         @record_accessor = record_accessor_create(@cmetrics_metric_key)
         @labels_accessor = record_accessor_create(@cmetrics_labels_key)
+        @fields_accessors = {}
+        conf.elements(name: "fields").each do |e|
+          e.each_pair{|k, _v|
+            e.has_key?(k) # Suppress unused warnings.
+            @fields_accessors[k] = record_accessor_create(k)
+          }
+        end
       end
 
-      def format_record_key_to_splunk_style(inner)
+      def format_to_splunk_style_with_dims(inner)
         subsystem = inner.delete("subsystem")
-        labels_str = if labels = @labels_accessor.call(inner)
-                       labels_str = labels.map {|k,v|
-                         if k == "cpu"
-                           "id.#{v}"
-                         else
-                           "#{k}.#{v}"
-                         end
-                       }.join(".")
-                     end
+        # labels will be treated as dimensions.
+        dimensions = Hash.new(0)
+        if labels = @labels_accessor.call(inner)
+          labels.map {|k,v|
+            dimensions[k] = v
+          }
+        end
         name = inner.delete("name")
-        [subsystem, labels_str, name].compact.reject{|e| e.empty?}.join(".")
+        return [subsystem, name].compact.reject{|e| e.empty?}.join("."), dimensions
       end
 
       def filter_stream(tag, es)
         new_es = Fluent::MultiEventStream.new
         es.each do |time, record|
           data = @record_accessor.call(record)
+          extra_fields = {}
+          @fields_accessors.each do |key, accessor|
+            extra_fields[key] = accessor.call(record)
+          end
           @serde.feed_each(data) do |cmetrics|
             metrics = cmetrics.metrics
             metrics.each do |metric|
               next if metric.empty?
 
               metric.each do |inner|
-                if @format_name_key_for_splunk_metric
-                  inner["name"] = format_record_key_to_splunk_style(inner)
+                if @format_to_splunk_metric
+                  inner["name"], dims = format_to_splunk_style_with_dims(inner)
+                  if @dimensions_key
+                    inner[@dimensions_key] = dims
+                  else
+                    inner.merge!(dims)
+                  end
+                end
+                if @fields_accessors
+                  inner.merge!(extra_fields)
                 end
                 time = Time.at(inner.delete("timestamp"))
                 new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)

data/lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb

@@ -0,0 +1,89 @@
+#
+# Copyright 2021- Calyptia Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/formatter"
+require 'fluent/plugin_helper/record_accessor'
+require 'fluent/plugin_helper'
+require "fluent/event"
+require "fluent/time"
+require "yajl"
+require "socket"
+
+module Fluent
+  module Plugin
+    class CMetricsSplunkMetricPayloadFormatter < Fluent::Plugin::Formatter
+      include PluginHelper::Mixin
+
+      Fluent::Plugin.register_formatter('cmetrics_splunk_metric_payload', self)
+
+      helpers :record_accessor
+
+      desc "cmetrics metrics name key"
+      config_param :cmetrics_name_key, :string, default: "name"
+      desc "cmetrics metrics value key"
+      config_param :cmetrics_value_key, :string, default: "value"
+      desc "cmetrics metrics dimensions key"
+      config_param :cmetrics_dims_key, :string, default: "dims"
+      desc "Specify host key"
+      config_param :host_key, :string, default: "host"
+      desc "Specify splunk index name"
+      config_param :index, :string, default: nil
+      desc "Specify splunk source name"
+      config_param :source, :string, default: nil
+      desc "Specify splunk sourcetype name"
+      config_param :sourcetype, :string, default: nil
+
+      def initialize
+        super
+        @default_host = Socket.gethostname
+      end
+
+      def configure(conf)
+        super
+
+        @cmetrics_name_accessor = record_accessor_create(@cmetrics_name_key)
+        @cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
+        @cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
+        @host_key_accessor = record_accessor_create(@host_key)
+      end
+
+      def format(tag, time, record)
+        host = if host = @host_key_accessor.call(record)
+                 host
+               else
+                 @default_host
+               end
+        payload = {
+          host: host,
+          # From the API reference
+          # https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+          time: time.to_f.to_s,
+          event: 'metric',
+        }
+        payload[:index] = @index if @index
+        payload[:source] = @source if @source
+        payload[:sourcetype] = @sourcetype if @sourcetype
+        fields = {
+          "metric_name:#{@cmetrics_name_accessor.call(record)}" => @cmetrics_value_accessor.call(record)
+        }
+        if dims = @cmetrics_dims_accessor.call(record)
+          fields.merge!(dims)
+        end
+        payload.merge!(fields)
+        Yajl.dump(payload)
+      end
+    end
+  end
+end

data/test/plugin/test_filter_cmetrics_parser.rb

@@ -1,21 +1,27 @@
 require "helper"
 require "fluent/plugin/filter_cmetrics_parser.rb"
+require 'socket'
 
 class CmetricsParserTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
   end
 
-  data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.id.10.mode.system.seconds_total"],
-       "filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum"],
-       "disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.device.nvme0n1.io_now"],
-       "network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.device.eth0.transmit_bytes_total"],
-      "none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5"])
-  test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key)|
+  data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.seconds_total", {"cpu" => "10", "mode" => "system"}],
+       "filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum", {}],
+       "disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.io_now", {"device" => "nvme0n1"}],
+       "network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.transmit_bytes_total", {"device" => "eth0"}],
+      "none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5", {}])
+  test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key, expected_dims)|
     json = Yajl.load(json_str)
-    d = create_driver(%[format_name_key_for_splunk_metric true])
-    assert_true d.instance.format_name_key_for_splunk_metric
-    assert_equal expected_format_key, d.instance.format_record_key_to_splunk_style(json)
+    d = create_driver(%[
+      format_to_splunk_metric true
+      dimensions_key dims
+    ])
+    assert_true d.instance.format_to_splunk_metric
+    formatted_key, dims = d.instance.format_to_splunk_style_with_dims(json)
+    assert_equal expected_format_key, formatted_key
+    assert_equal expected_dims, dims
   end
 
   sub_test_case "Actual filtering" do
@@ -24,13 +30,57 @@ class CmetricsParserTest < Test::Unit::TestCase
       @binary = File.read(@binary_path)
     end
 
-    test "#filter_stream" do
-      d = create_driver(%[format_name_key_for_splunk_metric true])
+    data("with dimensions" => "dims",
+         "without dimensions" => nil)
+    test "#filter_stream" do |data|
+      use_dimensions = data
+      d = if use_dimensions
+            create_driver(%[
+              format_to_splunk_metric true
+              dimensions_key dims
+            ])
+          else
+            create_driver(%[
+              format_to_splunk_metric true
+            ])
+          end
       time = event_time("2012-01-02 13:14:15")
       record = {"cmetrics" => @binary}
       d.run(default_tag: 'test') do
         d.feed(time, record)
       end
+      d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
+      assert do
+        d.filtered.size > 0
+      end
+    end
+
+    data("with dimensions" => "dims",
+         "without dimensions" => nil)
+    test "#filter_stream with host_key" do |data|
+      use_dimensions = data
+      d = if use_dimensions
+            create_driver(Fluent::Config::Element.new('ROOT', '', {
+                                                        "format_to_splunk_metric" => true,
+                                                        "dimensions_key" => "dims",
+                                                      }, [
+                                                        Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
+                                                      ]))
+          else
+            create_driver(Fluent::Config::Element.new('ROOT', '', {
+                                                        "format_to_splunk_metric" => true,
+                                                      }, [
+                                                        Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
+                                                      ]))
+          end
+      time = event_time("2012-01-02 13:14:15")
+      record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
+      d.run(default_tag: 'test') do
+        d.feed(time, record)
+      end
+      d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
       assert do
         d.filtered.size > 0
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cmetrics
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc3
+  version: 0.1.0.rc7
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-09 00:00:00.000000000 Z
+date: 2021-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,6 +100,7 @@ files:
 - Rakefile
 - fluent-plugin-cmetrics.gemspec
 - lib/fluent/plugin/filter_cmetrics_parser.rb
+- lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb
 - test/fixtures/cmetrics.bin
 - test/helper.rb
 - test/plugin/test_filter_cmetrics_parser.rb
@@ -122,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for cmetrics format handling.