fluent-plugin-cmetrics 0.1.0.rc2 → 0.1.0.rc6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa5328239515010bccb7710e1600b97fbfadf5674140056c4fda9434b65302e5
-  data.tar.gz: 2d0dd9208b1a9babc353fefa5cf2ffc4ae67b24b7aa18f63e402a76287198059
+  metadata.gz: 65548cae758782bc23022e7078dbab4a7c566b6175a652e7c5cee348a3b1ef93
+  data.tar.gz: e5fe401babbfe22ac5038130dd72d29135a794c155e4a294c59fbc4b5898dd41
 SHA512:
-  metadata.gz: 6eafe5b8ab643ab1b0f7e55d06fe7d5385321bf8689e8ed5d953f7fde04953b33acdff7a691d19be120a236253ae170a75ebef878025e291d83dfd3ce6a01d70
-  data.tar.gz: 5141e4794e95a066b951da679fb44f09fc61bc415ed6d960b16cb23eb2dbf2fa10692ce9aa29e2195e1ca55e63c5c9885e74aaf80d37e817594d742c6d87749b
+  metadata.gz: 6991cae004f796934cd0b4dd7d5d1ef9f175798c7c799ac0c417860451b6c4c04deb2d9c9fb0776a20eae180f826854b8c089b31ea79cdc2ea1ced376defe1a1
+  data.tar.gz: 28fd06e6a2df8d259aab80eff36a27cc2bd3a2155e47c45170c6d5704c24882d42226d3ff1f8f55fd8e6a73c1825adb45d1e05d46cffae0bea56e7b1097edd51

data/README.md

@@ -32,22 +32,69 @@ $ bundle
 
 ## Fluent::Plugin::CMetricsParserFilter
 
-### cmetric_metric_key (string) (optional)
+### cmetrics_metric_key (string) (optional)
 
 cmetrics metric key
 
 Default value: `cmetrics`.
 
-### cmetric_labels_key (string) (optional)
+### cmetrics_labels_key (string) (optional)
 
 cmetrics labels key
 
 Default value: `labels`.
 
-### format_name_key_for_splunk_metric (bool) (optional)
+### host_key (string) (optional)
 
-format name key for Splunk metrics
+hostname key
 
+Default value: `host`.
+
+### format_to_splunk_metric (bool) (optional)
+
+format to Splunk metrics
+
+### dimensions_key (string) (optional)
+
+dimensions key
+
+## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
+
+### cmetrics_name_key (string) (optional)
+
+cmetrics metrics name key
+
+Default value: `name`.
+
+### cmetrics_value_key (string) (optional)
+
+cmetrics metrics value key
+
+Default value: `value`.
+
+### cmetrics_dims_key (string) (optional)
+
+cmetrics metrics dimensions key
+
+Default value: `dims`.
+
+### host_key (string) (optional)
+
+Specify host key
+
+Default value: `host`.
+
+### index (string) (optional)
+
+Specify splunk index name
+
+### source (string) (optional)
+
+Specify splunk source name
+
+### sourcetype (string) (optional)
+
+Specify splunk sourcetype name
 
 ## Copyright
 

data/fluent-plugin-cmetrics.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-cmetrics"
-  spec.version = "0.1.0.rc2"
+  spec.version = "0.1.0.rc6"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["cosmo0920.oucc@gmail.com"]
 

data/lib/fluent/plugin/filter_cmetrics_parser.rb

@@ -30,43 +30,55 @@ module Fluent
       config_param :cmetrics_metric_key, :string, default: "cmetrics"
       desc "cmetrics labels key"
       config_param :cmetrics_labels_key, :string, default: "labels"
-      desc "format name key for Splunk metrics"
-      config_param :format_name_key_for_splunk_metric, :bool, default: false
+      desc "hostname key"
+      config_param :host_key, :string, default: "host"
+      desc "format to Splunk metrics"
+      config_param :format_to_splunk_metric, :bool, default: false
+      desc "dimensions key"
+      config_param :dimensions_key, :string, default: nil
 
       def configure(conf)
         super
         @serde = ::CMetrics::Serde.new
         @record_accessor = record_accessor_create(@cmetrics_metric_key)
         @labels_accessor = record_accessor_create(@cmetrics_labels_key)
+        @hostname_accessor = record_accessor_create(@host_key)
       end
 
-      def format_record_key_to_splunk_style(inner)
+      def format_to_splunk_style_with_dims(inner)
         subsystem = inner.delete("subsystem")
-        labels_str = if labels = @labels_accessor.call(inner)
-                       labels_str = labels.map {|k,v|
-                         if k == "cpu"
-                           "id_#{v}"
-                         else
-                           "#{k}_#{v}"
-                         end
-                       }.join("_")
-                     end
+        # labels will be treated as dimensions.
+        dimensions = Hash.new(0)
+        if labels = @labels_accessor.call(inner)
+          labels.map {|k,v|
+            dimensions[k] = v
+          }
+        end
         name = inner.delete("name")
-        [subsystem, labels_str, name].compact.reject{|e| e.empty?}.join("_")
+        return [subsystem, name].compact.reject{|e| e.empty?}.join("."), dimensions
       end
 
       def filter_stream(tag, es)
         new_es = Fluent::MultiEventStream.new
         es.each do |time, record|
           data = @record_accessor.call(record)
+          hostname = @hostname_accessor.call(record)
           @serde.feed_each(data) do |cmetrics|
             metrics = cmetrics.metrics
             metrics.each do |metric|
               next if metric.empty?
 
               metric.each do |inner|
-                if @format_name_key_for_splunk_metric
-                  inner["name"] = format_record_key_to_splunk_style(inner)
+                if @format_to_splunk_metric
+                  inner["name"], dims = format_to_splunk_style_with_dims(inner)
+                  if @dimensions_key
+                    inner[@dimensions_key] = dims
+                  else
+                    inner.merge!(dims)
+                  end
+                end
+                if hostname
+                  inner[@host_key] = hostname
                 end
                 time = Time.at(inner.delete("timestamp"))
                 new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)

data/lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb

@@ -0,0 +1,89 @@
+#
+# Copyright 2021- Calyptia Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/formatter"
+require 'fluent/plugin_helper/record_accessor'
+require 'fluent/plugin_helper'
+require "fluent/event"
+require "fluent/time"
+require "yajl"
+require "socket"
+
+module Fluent
+  module Plugin
+    class CMetricsSplunkMetricPayloadFormatter < Fluent::Plugin::Formatter
+      include PluginHelper::Mixin
+
+      Fluent::Plugin.register_formatter('cmetrics_splunk_metric_payload', self)
+
+      helpers :record_accessor
+
+      desc "cmetrics metrics name key"
+      config_param :cmetrics_name_key, :string, default: "name"
+      desc "cmetrics metrics value key"
+      config_param :cmetrics_value_key, :string, default: "value"
+      desc "cmetrics metrics dimensions key"
+      config_param :cmetrics_dims_key, :string, default: "dims"
+      desc "Specify host key"
+      config_param :host_key, :string, default: "host"
+      desc "Specify splunk index name"
+      config_param :index, :string, default: nil
+      desc "Specify splunk source name"
+      config_param :source, :string, default: nil
+      desc "Specify splunk sourcetype name"
+      config_param :sourcetype, :string, default: nil
+
+      def initialize
+        super
+        @default_host = Socket.gethostname
+      end
+
+      def configure(conf)
+        super
+
+        @cmetrics_name_accessor = record_accessor_create(@cmetrics_name_key)
+        @cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
+        @cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
+        @host_key_accessor = record_accessor_create(@host_key)
+      end
+
+      def format(tag, time, record)
+        host = if host = @host_key_accessor.call(record)
+                 host
+               else
+                 @default_host
+               end
+        payload = {
+          host: host,
+          # From the API reference
+          # https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+          time: time.to_f.to_s,
+          event: 'metric',
+        }
+        payload[:index] = @index if @index
+        payload[:source] = @source if @source
+        payload[:sourcetype] = @sourcetype if @sourcetype
+        fields = {
+          "metric_name:#{@cmetrics_name_accessor.call(record)}" => @cmetrics_value_accessor.call(record)
+        }
+        if dims = @cmetrics_dims_accessor.call(record)
+          fields.merge!(dims)
+        end
+        payload.merge!(fields)
+        Yajl.dump(payload)
+      end
+    end
+  end
+end

data/test/plugin/test_filter_cmetrics_parser.rb

@@ -1,21 +1,27 @@
 require "helper"
 require "fluent/plugin/filter_cmetrics_parser.rb"
+require 'socket'
 
 class CmetricsParserTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
   end
 
-  data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu_id_10_mode_system_seconds_total"],
-       "filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd_maximum"],
-       "disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk_device_nvme0n1_io_now"],
-       "network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network_device_eth0_transmit_bytes_total"],
-      "none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5"])
-  test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key)|
+  data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.seconds_total", {"cpu" => "10", "mode" => "system"}],
+       "filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum", {}],
+       "disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.io_now", {"device" => "nvme0n1"}],
+       "network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.transmit_bytes_total", {"device" => "eth0"}],
+      "none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5", {}])
+  test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key, expected_dims)|
     json = Yajl.load(json_str)
-    d = create_driver(%[format_name_key_for_splunk_metric true])
-    assert_true d.instance.format_name_key_for_splunk_metric
-    assert_equal expected_format_key, d.instance.format_record_key_to_splunk_style(json)
+    d = create_driver(%[
+      format_to_splunk_metric true
+      dimensions_key dims
+    ])
+    assert_true d.instance.format_to_splunk_metric
+    formatted_key, dims = d.instance.format_to_splunk_style_with_dims(json)
+    assert_equal expected_format_key, formatted_key
+    assert_equal expected_dims, dims
   end
 
   sub_test_case "Actual filtering" do
@@ -24,13 +30,55 @@ class CmetricsParserTest < Test::Unit::TestCase
       @binary = File.read(@binary_path)
     end
 
-    test "#filter_stream" do
-      d = create_driver(%[format_name_key_for_splunk_metric true])
+    data("with dimensions" => "dims",
+         "without dimensions" => nil)
+    test "#filter_stream" do |data|
+      use_dimensions = data
+      d = if use_dimensions
+            create_driver(%[
+              format_to_splunk_metric true
+              dimensions_key dims
+            ])
+          else
+            create_driver(%[
+              format_to_splunk_metric true
+            ])
+          end
       time = event_time("2012-01-02 13:14:15")
       record = {"cmetrics" => @binary}
       d.run(default_tag: 'test') do
         d.feed(time, record)
       end
+      d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
+      assert do
+        d.filtered.size > 0
+      end
+    end
+
+    data("with dimensions" => "dims",
+         "without dimensions" => nil)
+    test "#filter_stream with host_key" do |data|
+      use_dimensions = data
+      d = if use_dimensions
+            create_driver(%[
+              format_to_splunk_metric true
+              dimensions_key dims
+              host_key hostname
+            ])
+          else
+            create_driver(%[
+              format_to_splunk_metric true
+              host_key hostname
+            ])
+          end
+      time = event_time("2012-01-02 13:14:15")
+      record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
+      d.run(default_tag: 'test') do
+        d.feed(time, record)
+      end
+      d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
       assert do
         d.filtered.size > 0
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cmetrics
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc2
+  version: 0.1.0.rc6
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-08 00:00:00.000000000 Z
+date: 2021-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,6 +100,7 @@ files:
 - Rakefile
 - fluent-plugin-cmetrics.gemspec
 - lib/fluent/plugin/filter_cmetrics_parser.rb
+- lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb
 - test/fixtures/cmetrics.bin
 - test/helper.rb
 - test/plugin/test_filter_cmetrics_parser.rb
@@ -122,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for cmetrics format handling.