fluent-plugin-cmetrics 0.1.0.rc5 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 14dc765336d42503cd80ded601ff4c275a38cec0cc754b31f1f1a86d333520ac
4
- data.tar.gz: 89cac70e738c7b3a4d65189be8a78ab9cba97210864e38e8ef47bf11aa5c7940
3
+ metadata.gz: 24fd1261db862a55b33f49932214689ec9b65bdcd8aa0c3287993f21bba26943
4
+ data.tar.gz: c264c77b3be6894a9a36a5053b142ed8b27fb3fc89c6b2de74b36bc6ec407d01
5
5
  SHA512:
6
- metadata.gz: e11f1f3c56baba25406a0294e8c84458c26f15d1070b3f9a125838e9ac536386f32b705a5cbcbbddf242234a728be397bb0f12e5dd8e72c89a09917e4799bc69
7
- data.tar.gz: 28a10bb177b9758ac19a25abce5f04746051fb26e23842b6eef488b18c390e309ddba487656fb5df01c6975a0b34a4d5a829a1ec5add63be5f58b5fabb42ad36
6
+ metadata.gz: f0db025054c9c808875e29612dba17abccdb6fc6f3337cd41f2b68a2c6429b3133eb07cd8a4fb3e0a38f532a70f0c241bc597675f1767057db6fa8ca40f3d548
7
+ data.tar.gz: fa1764de2e06cc32c2d880366b98d3912224500e7214cd3cb75231813c09515b865279ec7e9a0ea639638968e1c624df2efec7c5706678c0f175a8719d63595e
data/README.md CHANGED
@@ -52,6 +52,26 @@ format to Splunk metrics
52
52
 
53
53
  dimensions key
54
54
 
55
+
56
+ ### \<fields\> section (optional) (single)
57
+
58
+ This secsion is used for adding extra fields into cmetrics msgpack payload parsed records.
59
+
60
+ For example, the following configuration should add hostname records into parsed records:
61
+
62
+ ```aconf
63
+ <filter super.awesome.tag.**>
64
+ @type cmetrics_parser
65
+ format_to_splunk_metric true
66
+ dimensions_key dims
67
+ <fields>
68
+ hostname
69
+ </fields>
70
+ </filter>
71
+ ```
72
+
73
+ On later data pipeline, `hostname` key can be used as some additional work.
74
+
55
75
  ## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
56
76
 
57
77
  ### cmetrics_name_key (string) (optional)
@@ -90,6 +110,16 @@ Specify splunk source name
90
110
 
91
111
  Specify splunk sourcetype name
92
112
 
113
+ ### only_use_last_field_keys (bool) (optional)
114
+
115
+ Whether using only last key values or not when processing for field keys
116
+
117
+ Default value: `false`
118
+
119
+ ### \<fields\> section (optional) (single)
120
+
121
+ This secsion is used for adding extra fields as dimensions into cmetrics msgpack Splunk metric payload.
122
+
93
123
  ## Copyright
94
124
 
95
125
  * Copyright(c) 2021- Calyptia Inc.
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-cmetrics"
6
- spec.version = "0.1.0.rc5"
6
+ spec.version = "0.1.0"
7
7
  spec.authors = ["Hiroshi Hatake"]
8
8
  spec.email = ["cosmo0920.oucc@gmail.com"]
9
9
 
@@ -34,12 +34,23 @@ module Fluent
34
34
  config_param :format_to_splunk_metric, :bool, default: false
35
35
  desc "dimensions key"
36
36
  config_param :dimensions_key, :string, default: nil
37
+ desc "Add additional records for Splunk dimensions"
38
+ config_section :fields, init: false, multi: false,required: false do
39
+ # Nothing here. For later purpose.
40
+ end
37
41
 
38
42
  def configure(conf)
39
43
  super
40
44
  @serde = ::CMetrics::Serde.new
41
45
  @record_accessor = record_accessor_create(@cmetrics_metric_key)
42
46
  @labels_accessor = record_accessor_create(@cmetrics_labels_key)
47
+ @fields_accessors = {}
48
+ conf.elements(name: "fields").each do |e|
49
+ e.each_pair{|k, _v|
50
+ e.has_key?(k) # Suppress unused warnings.
51
+ @fields_accessors[k] = record_accessor_create(k)
52
+ }
53
+ end
43
54
  end
44
55
 
45
56
  def format_to_splunk_style_with_dims(inner)
@@ -59,6 +70,10 @@ module Fluent
59
70
  new_es = Fluent::MultiEventStream.new
60
71
  es.each do |time, record|
61
72
  data = @record_accessor.call(record)
73
+ extra_fields = {}
74
+ @fields_accessors.each do |key, accessor|
75
+ extra_fields[key] = accessor.call(record)
76
+ end
62
77
  @serde.feed_each(data) do |cmetrics|
63
78
  metrics = cmetrics.metrics
64
79
  metrics.each do |metric|
@@ -73,6 +88,9 @@ module Fluent
73
88
  inner.merge!(dims)
74
89
  end
75
90
  end
91
+ if @fields_accessors
92
+ inner.merge!(extra_fields)
93
+ end
76
94
  time = Time.at(inner.delete("timestamp"))
77
95
  new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)
78
96
  end
@@ -44,6 +44,11 @@ module Fluent
44
44
  config_param :source, :string, default: nil
45
45
  desc "Specify splunk sourcetype name"
46
46
  config_param :sourcetype, :string, default: nil
47
+ desc "Whether using only last key values or not when processing for field keys"
48
+ config_param :only_use_last_field_keys, :bool, default: false
49
+ config_section :fields, init: false, multi: false, required: false do
50
+ # Other parameters nothing here. For later purpose.
51
+ end
47
52
 
48
53
  def initialize
49
54
  super
@@ -57,6 +62,13 @@ module Fluent
57
62
  @cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
58
63
  @cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
59
64
  @host_key_accessor = record_accessor_create(@host_key)
65
+ @fields_accessors = {}
66
+ conf.elements(name: "fields").each do |e|
67
+ e.each_pair{|k, _v|
68
+ e.has_key?(k) # Suppress unused warnings.
69
+ @fields_accessors[k] = record_accessor_create(k)
70
+ }
71
+ end
60
72
  end
61
73
 
62
74
  def format(tag, time, record)
@@ -65,6 +77,8 @@ module Fluent
65
77
  else
66
78
  @default_host
67
79
  end
80
+ extra_fields = {}
81
+
68
82
  payload = {
69
83
  host: host,
70
84
  # From the API reference
@@ -81,6 +95,18 @@ module Fluent
81
95
  if dims = @cmetrics_dims_accessor.call(record)
82
96
  fields.merge!(dims)
83
97
  end
98
+ @fields_accessors.each do |key, accessor|
99
+ if record = accessor.call(record)
100
+ record.each do |k, v|
101
+ if @only_use_last_field_keys
102
+ extra_fields[k.split(".").last] = v
103
+ else
104
+ extra_fields[k] = v
105
+ end
106
+ end
107
+ end
108
+ fields.merge!(extra_fields)
109
+ end
84
110
  payload.merge!(fields)
85
111
  Yajl.dump(payload)
86
112
  end
@@ -1,5 +1,6 @@
1
1
  require "helper"
2
2
  require "fluent/plugin/filter_cmetrics_parser.rb"
3
+ require 'socket'
3
4
 
4
5
  class CmetricsParserTest < Test::Unit::TestCase
5
6
  setup do
@@ -49,6 +50,37 @@ class CmetricsParserTest < Test::Unit::TestCase
49
50
  d.feed(time, record)
50
51
  end
51
52
  d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
53
+ d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
54
+ assert do
55
+ d.filtered.size > 0
56
+ end
57
+ end
58
+
59
+ data("with dimensions" => "dims",
60
+ "without dimensions" => nil)
61
+ test "#filter_stream with host_key" do |data|
62
+ use_dimensions = data
63
+ d = if use_dimensions
64
+ create_driver(Fluent::Config::Element.new('ROOT', '', {
65
+ "format_to_splunk_metric" => true,
66
+ "dimensions_key" => "dims",
67
+ }, [
68
+ Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
69
+ ]))
70
+ else
71
+ create_driver(Fluent::Config::Element.new('ROOT', '', {
72
+ "format_to_splunk_metric" => true,
73
+ }, [
74
+ Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
75
+ ]))
76
+ end
77
+ time = event_time("2012-01-02 13:14:15")
78
+ record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
79
+ d.run(default_tag: 'test') do
80
+ d.feed(time, record)
81
+ end
82
+ d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
83
+ d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
52
84
  assert do
53
85
  d.filtered.size > 0
54
86
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cmetrics
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0.rc5
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hiroshi Hatake
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-09-24 00:00:00.000000000 Z
11
+ date: 2021-12-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -119,9 +119,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
119
119
  version: '0'
120
120
  required_rubygems_version: !ruby/object:Gem::Requirement
121
121
  requirements:
122
- - - ">"
122
+ - - ">="
123
123
  - !ruby/object:Gem::Version
124
- version: 1.3.1
124
+ version: '0'
125
125
  requirements: []
126
126
  rubygems_version: 3.2.22
127
127
  signing_key: