fluent-plugin-cmetrics 0.1.0.rc5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14dc765336d42503cd80ded601ff4c275a38cec0cc754b31f1f1a86d333520ac
-  data.tar.gz: 89cac70e738c7b3a4d65189be8a78ab9cba97210864e38e8ef47bf11aa5c7940
+  metadata.gz: 24fd1261db862a55b33f49932214689ec9b65bdcd8aa0c3287993f21bba26943
+  data.tar.gz: c264c77b3be6894a9a36a5053b142ed8b27fb3fc89c6b2de74b36bc6ec407d01
 SHA512:
-  metadata.gz: e11f1f3c56baba25406a0294e8c84458c26f15d1070b3f9a125838e9ac536386f32b705a5cbcbbddf242234a728be397bb0f12e5dd8e72c89a09917e4799bc69
-  data.tar.gz: 28a10bb177b9758ac19a25abce5f04746051fb26e23842b6eef488b18c390e309ddba487656fb5df01c6975a0b34a4d5a829a1ec5add63be5f58b5fabb42ad36
+  metadata.gz: f0db025054c9c808875e29612dba17abccdb6fc6f3337cd41f2b68a2c6429b3133eb07cd8a4fb3e0a38f532a70f0c241bc597675f1767057db6fa8ca40f3d548
+  data.tar.gz: fa1764de2e06cc32c2d880366b98d3912224500e7214cd3cb75231813c09515b865279ec7e9a0ea639638968e1c624df2efec7c5706678c0f175a8719d63595e

data/README.md

@@ -52,6 +52,26 @@ format to Splunk metrics
 
 dimensions key
 
+
+### \<fields\> section (optional) (single)
+
+This secsion is used for adding extra fields into cmetrics msgpack payload parsed records.
+
+For example, the following configuration should add hostname records into parsed records:
+
+```aconf
+<filter super.awesome.tag.**>
+  @type cmetrics_parser
+  format_to_splunk_metric true
+  dimensions_key dims
+  <fields>
+    hostname
+  </fields>
+</filter>
+```
+
+On later data pipeline, `hostname` key can be used as some additional work.
+
 ## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
 
 ### cmetrics_name_key (string) (optional)
@@ -90,6 +110,16 @@ Specify splunk source name
 
 Specify splunk sourcetype name
 
+### only_use_last_field_keys (bool) (optional)
+
+Whether using only last key values or not when processing for field keys
+
+Default value: `false`
+
+### \<fields\> section (optional) (single)
+
+This secsion is used for adding extra fields as dimensions into cmetrics msgpack Splunk metric payload.
+
 ## Copyright
 
 * Copyright(c) 2021- Calyptia Inc.

data/fluent-plugin-cmetrics.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-cmetrics"
-  spec.version = "0.1.0.rc5"
+  spec.version = "0.1.0"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["cosmo0920.oucc@gmail.com"]
 

data/lib/fluent/plugin/filter_cmetrics_parser.rb

@@ -34,12 +34,23 @@ module Fluent
       config_param :format_to_splunk_metric, :bool, default: false
       desc "dimensions key"
       config_param :dimensions_key, :string, default: nil
+      desc "Add additional records for Splunk dimensions"
+      config_section :fields, init: false, multi: false,required: false do
+        # Nothing here. For later purpose.
+      end
 
       def configure(conf)
         super
         @serde = ::CMetrics::Serde.new
         @record_accessor = record_accessor_create(@cmetrics_metric_key)
         @labels_accessor = record_accessor_create(@cmetrics_labels_key)
+        @fields_accessors = {}
+        conf.elements(name: "fields").each do |e|
+          e.each_pair{|k, _v|
+            e.has_key?(k) # Suppress unused warnings.
+            @fields_accessors[k] = record_accessor_create(k)
+          }
+        end
       end
 
       def format_to_splunk_style_with_dims(inner)
@@ -59,6 +70,10 @@ module Fluent
         new_es = Fluent::MultiEventStream.new
         es.each do |time, record|
           data = @record_accessor.call(record)
+          extra_fields = {}
+          @fields_accessors.each do |key, accessor|
+            extra_fields[key] = accessor.call(record)
+          end
           @serde.feed_each(data) do |cmetrics|
             metrics = cmetrics.metrics
             metrics.each do |metric|
@@ -73,6 +88,9 @@ module Fluent
                     inner.merge!(dims)
                   end
                 end
+                if @fields_accessors
+                  inner.merge!(extra_fields)
+                end
                 time = Time.at(inner.delete("timestamp"))
                 new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)
               end

data/lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb

@@ -44,6 +44,11 @@ module Fluent
       config_param :source, :string, default: nil
       desc "Specify splunk sourcetype name"
       config_param :sourcetype, :string, default: nil
+      desc "Whether using only last key values or not when processing for field keys"
+      config_param :only_use_last_field_keys, :bool, default: false
+      config_section :fields, init: false, multi: false, required: false do
+        # Other parameters nothing here. For later purpose.
+      end
 
       def initialize
         super
@@ -57,6 +62,13 @@ module Fluent
         @cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
         @cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
         @host_key_accessor = record_accessor_create(@host_key)
+        @fields_accessors = {}
+        conf.elements(name: "fields").each do |e|
+          e.each_pair{|k, _v|
+            e.has_key?(k) # Suppress unused warnings.
+            @fields_accessors[k] = record_accessor_create(k)
+          }
+        end
       end
 
       def format(tag, time, record)
@@ -65,6 +77,8 @@ module Fluent
                else
                  @default_host
                end
+        extra_fields = {}
+
         payload = {
           host: host,
           # From the API reference
@@ -81,6 +95,18 @@ module Fluent
         if dims = @cmetrics_dims_accessor.call(record)
           fields.merge!(dims)
         end
+        @fields_accessors.each do |key, accessor|
+          if record = accessor.call(record)
+            record.each do |k, v|
+              if @only_use_last_field_keys
+                extra_fields[k.split(".").last] = v
+              else
+                extra_fields[k] = v
+              end
+            end
+          end
+          fields.merge!(extra_fields)
+        end
         payload.merge!(fields)
         Yajl.dump(payload)
       end

data/test/plugin/test_filter_cmetrics_parser.rb

@@ -1,5 +1,6 @@
 require "helper"
 require "fluent/plugin/filter_cmetrics_parser.rb"
+require 'socket'
 
 class CmetricsParserTest < Test::Unit::TestCase
   setup do
@@ -49,6 +50,37 @@ class CmetricsParserTest < Test::Unit::TestCase
         d.feed(time, record)
       end
       d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
+      assert do
+        d.filtered.size > 0
+      end
+    end
+
+    data("with dimensions" => "dims",
+         "without dimensions" => nil)
+    test "#filter_stream with host_key" do |data|
+      use_dimensions = data
+      d = if use_dimensions
+            create_driver(Fluent::Config::Element.new('ROOT', '', {
+                                                        "format_to_splunk_metric" => true,
+                                                        "dimensions_key" => "dims",
+                                                      }, [
+                                                        Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
+                                                      ]))
+          else
+            create_driver(Fluent::Config::Element.new('ROOT', '', {
+                                                        "format_to_splunk_metric" => true,
+                                                      }, [
+                                                        Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
+                                                      ]))
+          end
+      time = event_time("2012-01-02 13:14:15")
+      record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
+      d.run(default_tag: 'test') do
+        d.feed(time, record)
+      end
+      d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
+      d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
       assert do
         d.filtered.size > 0
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cmetrics
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc5
+  version: 0.1.0
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-24 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -119,9 +119,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.2.22
 signing_key: