fluent-plugin-cmetrics 0.1.0.rc → 0.1.0.rc5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69a99953b806c28c73bf3e734c2419e3ce5df8974e6bdd12a93d0e0d51096ceb
-  data.tar.gz: bfe77f7b81c5f10ce5657bd5ea24b30c29f2ebd9b62fe5d10f82988d06f2e85c
+  metadata.gz: 14dc765336d42503cd80ded601ff4c275a38cec0cc754b31f1f1a86d333520ac
+  data.tar.gz: 89cac70e738c7b3a4d65189be8a78ab9cba97210864e38e8ef47bf11aa5c7940
 SHA512:
-  metadata.gz: 82b2b4e6bcc4f95c02e4eabfe000f6010f8e2ee07a7b7491057e30d39fc2bb9cc85555a3f7c9da0a60a901f2faba10882694ee1c08e8bf928614140302b330a6
-  data.tar.gz: ab04d2523a2ed3bec53859b419b0ac77d25fd27dfbc7462fa7ece07bb8b63cd0f8889cd89d027414b9713f2f3f30caad8d9a96eb3e0015cd6a324dc19b062bee
+  metadata.gz: e11f1f3c56baba25406a0294e8c84458c26f15d1070b3f9a125838e9ac536386f32b705a5cbcbbddf242234a728be397bb0f12e5dd8e72c89a09917e4799bc69
+  data.tar.gz: 28a10bb177b9758ac19a25abce5f04746051fb26e23842b6eef488b18c390e309ddba487656fb5df01c6975a0b34a4d5a829a1ec5add63be5f58b5fabb42ad36

data/README.md

@@ -1,8 +1,6 @@
 # fluent-plugin-cmetrics
 
-[Fluentd](https://fluentd.org/) parser plugin to do something.
-
-TODO: write description for you plugin.
+[Fluentd](https://fluentd.org/) plugin for cmetrics format handling.
 
 ## Installation
 
@@ -26,15 +24,71 @@ And then execute:
 $ bundle
 ```
 
-## Configuration
+## Plugin helpers
 
-You can generate configuration template:
+* [record_accessor](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-record_accessor)
 
-```
-$ fluent-plugin-config-format parser cmetrics
-```
+* See also: [Filter Plugin Overview](https://docs.fluentd.org/v/1.0/filter#overview)
+
+## Fluent::Plugin::CMetricsParserFilter
+
+### cmetrics_metric_key (string) (optional)
+
+cmetrics metric key
+
+Default value: `cmetrics`.
+
+### cmetrics_labels_key (string) (optional)
+
+cmetrics labels key
+
+Default value: `labels`.
+
+### format_to_splunk_metric (bool) (optional)
+
+format to Splunk metrics
+
+### dimensions_key (string) (optional)
+
+dimensions key
+
+## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
+
+### cmetrics_name_key (string) (optional)
+
+cmetrics metrics name key
+
+Default value: `name`.
+
+### cmetrics_value_key (string) (optional)
+
+cmetrics metrics value key
+
+Default value: `value`.
+
+### cmetrics_dims_key (string) (optional)
+
+cmetrics metrics dimensions key
+
+Default value: `dims`.
+
+### host_key (string) (optional)
+
+Specify host key
+
+Default value: `host`.
+
+### index (string) (optional)
+
+Specify splunk index name
+
+### source (string) (optional)
+
+Specify splunk source name
+
+### sourcetype (string) (optional)
 
-You can copy and paste generated documents here.
+Specify splunk sourcetype name
 
 ## Copyright
 

data/fluent-plugin-cmetrics.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-cmetrics"
-  spec.version = "0.1.0.rc"
+  spec.version = "0.1.0.rc5"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["cosmo0920.oucc@gmail.com"]
 

data/lib/fluent/plugin/filter_cmetrics_parser.rb

@@ -27,17 +27,32 @@ module Fluent
       helpers :record_accessor
 
       desc "cmetrics metric key"
-      config_param :cmetric_metric_key, :string, default: "cmetric"
+      config_param :cmetrics_metric_key, :string, default: "cmetrics"
       desc "cmetrics labels key"
-      config_param :cmetric_labels_key, :string, default: "labels"
-      desc "format name key for Splunk metrics"
-      config_param :format_name_key_for_splunk_metric, :bool, default: false
+      config_param :cmetrics_labels_key, :string, default: "labels"
+      desc "format to Splunk metrics"
+      config_param :format_to_splunk_metric, :bool, default: false
+      desc "dimensions key"
+      config_param :dimensions_key, :string, default: nil
 
       def configure(conf)
         super
         @serde = ::CMetrics::Serde.new
-        @record_accessor = record_accessor_create(@cmetric_metric_key)
-        @labels_accessor = record_accessor_create(@cmetric_labels_key)
+        @record_accessor = record_accessor_create(@cmetrics_metric_key)
+        @labels_accessor = record_accessor_create(@cmetrics_labels_key)
+      end
+
+      def format_to_splunk_style_with_dims(inner)
+        subsystem = inner.delete("subsystem")
+        # labels will be treated as dimensions.
+        dimensions = Hash.new(0)
+        if labels = @labels_accessor.call(inner)
+          labels.map {|k,v|
+            dimensions[k] = v
+          }
+        end
+        name = inner.delete("name")
+        return [subsystem, name].compact.reject{|e| e.empty?}.join("."), dimensions
       end
 
       def filter_stream(tag, es)
@@ -50,10 +65,13 @@ module Fluent
               next if metric.empty?
 
               metric.each do |inner|
-                if @format_name_key_for_splunk_metric && labels = @labels_accessor.call(inner)
-                  name = inner.delete("name")
-                  labels_str = labels.map {|k,v| "#{k}_#{v}"}.join("_")
-                  inner["name"] = "#{labels_str}_#{name}"
+                if @format_to_splunk_metric
+                  inner["name"], dims = format_to_splunk_style_with_dims(inner)
+                  if @dimensions_key
+                    inner[@dimensions_key] = dims
+                  else
+                    inner.merge!(dims)
+                  end
                 end
                 time = Time.at(inner.delete("timestamp"))
                 new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)

data/lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb

@@ -0,0 +1,89 @@
+#
+# Copyright 2021- Calyptia Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/formatter"
+require 'fluent/plugin_helper/record_accessor'
+require 'fluent/plugin_helper'
+require "fluent/event"
+require "fluent/time"
+require "yajl"
+require "socket"
+
+module Fluent
+  module Plugin
+    class CMetricsSplunkMetricPayloadFormatter < Fluent::Plugin::Formatter
+      include PluginHelper::Mixin
+
+      Fluent::Plugin.register_formatter('cmetrics_splunk_metric_payload', self)
+
+      helpers :record_accessor
+
+      desc "cmetrics metrics name key"
+      config_param :cmetrics_name_key, :string, default: "name"
+      desc "cmetrics metrics value key"
+      config_param :cmetrics_value_key, :string, default: "value"
+      desc "cmetrics metrics dimensions key"
+      config_param :cmetrics_dims_key, :string, default: "dims"
+      desc "Specify host key"
+      config_param :host_key, :string, default: "host"
+      desc "Specify splunk index name"
+      config_param :index, :string, default: nil
+      desc "Specify splunk source name"
+      config_param :source, :string, default: nil
+      desc "Specify splunk sourcetype name"
+      config_param :sourcetype, :string, default: nil
+
+      def initialize
+        super
+        @default_host = Socket.gethostname
+      end
+
+      def configure(conf)
+        super
+
+        @cmetrics_name_accessor = record_accessor_create(@cmetrics_name_key)
+        @cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
+        @cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
+        @host_key_accessor = record_accessor_create(@host_key)
+      end
+
+      def format(tag, time, record)
+        host = if host = @host_key_accessor.call(record)
+                 host
+               else
+                 @default_host
+               end
+        payload = {
+          host: host,
+          # From the API reference
+          # https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+          time: time.to_f.to_s,
+          event: 'metric',
+        }
+        payload[:index] = @index if @index
+        payload[:source] = @source if @source
+        payload[:sourcetype] = @sourcetype if @sourcetype
+        fields = {
+          "metric_name:#{@cmetrics_name_accessor.call(record)}" => @cmetrics_value_accessor.call(record)
+        }
+        if dims = @cmetrics_dims_accessor.call(record)
+          fields.merge!(dims)
+        end
+        payload.merge!(fields)
+        Yajl.dump(payload)
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -1,7 +1,7 @@
 $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
 require "test-unit"
 require "fluent/test"
-require "fluent/test/driver/parser"
+require "fluent/test/driver/filter"
 require "fluent/test/helpers"
 
 Test::Unit::TestCase.include(Fluent::Test::Helpers)

data/test/plugin/test_filter_cmetrics_parser.rb

@@ -6,8 +6,53 @@ class CmetricsParserTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  test "failure" do
-    flunk
+  data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.seconds_total", {"cpu" => "10", "mode" => "system"}],
+       "filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum", {}],
+       "disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.io_now", {"device" => "nvme0n1"}],
+       "network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.transmit_bytes_total", {"device" => "eth0"}],
+      "none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5", {}])
+  test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key, expected_dims)|
+    json = Yajl.load(json_str)
+    d = create_driver(%[
+      format_to_splunk_metric true
+      dimensions_key dims
+    ])
+    assert_true d.instance.format_to_splunk_metric
+    formatted_key, dims = d.instance.format_to_splunk_style_with_dims(json)
+    assert_equal expected_format_key, formatted_key
+    assert_equal expected_dims, dims
+  end
+
+  sub_test_case "Actual filtering" do
+    setup do
+      @binary_path = File.join(File.dirname(__dir__), "fixtures", "cmetrics.bin")
+      @binary = File.read(@binary_path)
+    end
+
+    data("with dimensions" => "dims",
+         "without dimensions" => nil)
+    test "#filter_stream" do |data|
+      use_dimensions = data
+      d = if use_dimensions
+            create_driver(%[
+              format_to_splunk_metric true
+              dimensions_key dims
+            ])
+          else
+            create_driver(%[
+              format_to_splunk_metric true
+            ])
+          end
+      time = event_time("2012-01-02 13:14:15")
+      record = {"cmetrics" => @binary}
+      d.run(default_tag: 'test') do
+        d.feed(time, record)
+      end
+      d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      assert do
+        d.filtered.size > 0
+      end
+    end
   end
 
   private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cmetrics
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc
+  version: 0.1.0.rc5
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-24 00:00:00.000000000 Z
+date: 2021-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,6 +100,8 @@ files:
 - Rakefile
 - fluent-plugin-cmetrics.gemspec
 - lib/fluent/plugin/filter_cmetrics_parser.rb
+- lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb
+- test/fixtures/cmetrics.bin
 - test/helper.rb
 - test/plugin/test_filter_cmetrics_parser.rb
 homepage: https://github.com/calyptia/fluent-plugin-cmetrics
@@ -126,5 +128,6 @@ signing_key:
 specification_version: 4
 summary: Fluentd plugin for cmetrics format handling.
 test_files:
+- test/fixtures/cmetrics.bin
 - test/helper.rb
 - test/plugin/test_filter_cmetrics_parser.rb