fluent-plugin-cmetrics 0.1.0.rc5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24fd1261db862a55b33f49932214689ec9b65bdcd8aa0c3287993f21bba26943
|
4
|
+
data.tar.gz: c264c77b3be6894a9a36a5053b142ed8b27fb3fc89c6b2de74b36bc6ec407d01
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0db025054c9c808875e29612dba17abccdb6fc6f3337cd41f2b68a2c6429b3133eb07cd8a4fb3e0a38f532a70f0c241bc597675f1767057db6fa8ca40f3d548
|
7
|
+
data.tar.gz: fa1764de2e06cc32c2d880366b98d3912224500e7214cd3cb75231813c09515b865279ec7e9a0ea639638968e1c624df2efec7c5706678c0f175a8719d63595e
|
data/README.md
CHANGED
@@ -52,6 +52,26 @@ format to Splunk metrics
|
|
52
52
|
|
53
53
|
dimensions key
|
54
54
|
|
55
|
+
|
56
|
+
### \<fields\> section (optional) (single)
|
57
|
+
|
58
|
+
This secsion is used for adding extra fields into cmetrics msgpack payload parsed records.
|
59
|
+
|
60
|
+
For example, the following configuration should add hostname records into parsed records:
|
61
|
+
|
62
|
+
```aconf
|
63
|
+
<filter super.awesome.tag.**>
|
64
|
+
@type cmetrics_parser
|
65
|
+
format_to_splunk_metric true
|
66
|
+
dimensions_key dims
|
67
|
+
<fields>
|
68
|
+
hostname
|
69
|
+
</fields>
|
70
|
+
</filter>
|
71
|
+
```
|
72
|
+
|
73
|
+
On later data pipeline, `hostname` key can be used as some additional work.
|
74
|
+
|
55
75
|
## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
|
56
76
|
|
57
77
|
### cmetrics_name_key (string) (optional)
|
@@ -90,6 +110,16 @@ Specify splunk source name
|
|
90
110
|
|
91
111
|
Specify splunk sourcetype name
|
92
112
|
|
113
|
+
### only_use_last_field_keys (bool) (optional)
|
114
|
+
|
115
|
+
Whether using only last key values or not when processing for field keys
|
116
|
+
|
117
|
+
Default value: `false`
|
118
|
+
|
119
|
+
### \<fields\> section (optional) (single)
|
120
|
+
|
121
|
+
This secsion is used for adding extra fields as dimensions into cmetrics msgpack Splunk metric payload.
|
122
|
+
|
93
123
|
## Copyright
|
94
124
|
|
95
125
|
* Copyright(c) 2021- Calyptia Inc.
|
@@ -34,12 +34,23 @@ module Fluent
|
|
34
34
|
config_param :format_to_splunk_metric, :bool, default: false
|
35
35
|
desc "dimensions key"
|
36
36
|
config_param :dimensions_key, :string, default: nil
|
37
|
+
desc "Add additional records for Splunk dimensions"
|
38
|
+
config_section :fields, init: false, multi: false,required: false do
|
39
|
+
# Nothing here. For later purpose.
|
40
|
+
end
|
37
41
|
|
38
42
|
def configure(conf)
|
39
43
|
super
|
40
44
|
@serde = ::CMetrics::Serde.new
|
41
45
|
@record_accessor = record_accessor_create(@cmetrics_metric_key)
|
42
46
|
@labels_accessor = record_accessor_create(@cmetrics_labels_key)
|
47
|
+
@fields_accessors = {}
|
48
|
+
conf.elements(name: "fields").each do |e|
|
49
|
+
e.each_pair{|k, _v|
|
50
|
+
e.has_key?(k) # Suppress unused warnings.
|
51
|
+
@fields_accessors[k] = record_accessor_create(k)
|
52
|
+
}
|
53
|
+
end
|
43
54
|
end
|
44
55
|
|
45
56
|
def format_to_splunk_style_with_dims(inner)
|
@@ -59,6 +70,10 @@ module Fluent
|
|
59
70
|
new_es = Fluent::MultiEventStream.new
|
60
71
|
es.each do |time, record|
|
61
72
|
data = @record_accessor.call(record)
|
73
|
+
extra_fields = {}
|
74
|
+
@fields_accessors.each do |key, accessor|
|
75
|
+
extra_fields[key] = accessor.call(record)
|
76
|
+
end
|
62
77
|
@serde.feed_each(data) do |cmetrics|
|
63
78
|
metrics = cmetrics.metrics
|
64
79
|
metrics.each do |metric|
|
@@ -73,6 +88,9 @@ module Fluent
|
|
73
88
|
inner.merge!(dims)
|
74
89
|
end
|
75
90
|
end
|
91
|
+
if @fields_accessors
|
92
|
+
inner.merge!(extra_fields)
|
93
|
+
end
|
76
94
|
time = Time.at(inner.delete("timestamp"))
|
77
95
|
new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)
|
78
96
|
end
|
@@ -44,6 +44,11 @@ module Fluent
|
|
44
44
|
config_param :source, :string, default: nil
|
45
45
|
desc "Specify splunk sourcetype name"
|
46
46
|
config_param :sourcetype, :string, default: nil
|
47
|
+
desc "Whether using only last key values or not when processing for field keys"
|
48
|
+
config_param :only_use_last_field_keys, :bool, default: false
|
49
|
+
config_section :fields, init: false, multi: false, required: false do
|
50
|
+
# Other parameters nothing here. For later purpose.
|
51
|
+
end
|
47
52
|
|
48
53
|
def initialize
|
49
54
|
super
|
@@ -57,6 +62,13 @@ module Fluent
|
|
57
62
|
@cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
|
58
63
|
@cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
|
59
64
|
@host_key_accessor = record_accessor_create(@host_key)
|
65
|
+
@fields_accessors = {}
|
66
|
+
conf.elements(name: "fields").each do |e|
|
67
|
+
e.each_pair{|k, _v|
|
68
|
+
e.has_key?(k) # Suppress unused warnings.
|
69
|
+
@fields_accessors[k] = record_accessor_create(k)
|
70
|
+
}
|
71
|
+
end
|
60
72
|
end
|
61
73
|
|
62
74
|
def format(tag, time, record)
|
@@ -65,6 +77,8 @@ module Fluent
|
|
65
77
|
else
|
66
78
|
@default_host
|
67
79
|
end
|
80
|
+
extra_fields = {}
|
81
|
+
|
68
82
|
payload = {
|
69
83
|
host: host,
|
70
84
|
# From the API reference
|
@@ -81,6 +95,18 @@ module Fluent
|
|
81
95
|
if dims = @cmetrics_dims_accessor.call(record)
|
82
96
|
fields.merge!(dims)
|
83
97
|
end
|
98
|
+
@fields_accessors.each do |key, accessor|
|
99
|
+
if record = accessor.call(record)
|
100
|
+
record.each do |k, v|
|
101
|
+
if @only_use_last_field_keys
|
102
|
+
extra_fields[k.split(".").last] = v
|
103
|
+
else
|
104
|
+
extra_fields[k] = v
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
fields.merge!(extra_fields)
|
109
|
+
end
|
84
110
|
payload.merge!(fields)
|
85
111
|
Yajl.dump(payload)
|
86
112
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require "helper"
|
2
2
|
require "fluent/plugin/filter_cmetrics_parser.rb"
|
3
|
+
require 'socket'
|
3
4
|
|
4
5
|
class CmetricsParserTest < Test::Unit::TestCase
|
5
6
|
setup do
|
@@ -49,6 +50,37 @@ class CmetricsParserTest < Test::Unit::TestCase
|
|
49
50
|
d.feed(time, record)
|
50
51
|
end
|
51
52
|
d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
|
53
|
+
d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
|
54
|
+
assert do
|
55
|
+
d.filtered.size > 0
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
data("with dimensions" => "dims",
|
60
|
+
"without dimensions" => nil)
|
61
|
+
test "#filter_stream with host_key" do |data|
|
62
|
+
use_dimensions = data
|
63
|
+
d = if use_dimensions
|
64
|
+
create_driver(Fluent::Config::Element.new('ROOT', '', {
|
65
|
+
"format_to_splunk_metric" => true,
|
66
|
+
"dimensions_key" => "dims",
|
67
|
+
}, [
|
68
|
+
Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
|
69
|
+
]))
|
70
|
+
else
|
71
|
+
create_driver(Fluent::Config::Element.new('ROOT', '', {
|
72
|
+
"format_to_splunk_metric" => true,
|
73
|
+
}, [
|
74
|
+
Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
|
75
|
+
]))
|
76
|
+
end
|
77
|
+
time = event_time("2012-01-02 13:14:15")
|
78
|
+
record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
|
79
|
+
d.run(default_tag: 'test') do
|
80
|
+
d.feed(time, record)
|
81
|
+
end
|
82
|
+
d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
|
83
|
+
d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
|
52
84
|
assert do
|
53
85
|
d.filtered.size > 0
|
54
86
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cmetrics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroshi Hatake
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-12-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -119,9 +119,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
119
119
|
version: '0'
|
120
120
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- - "
|
122
|
+
- - ">="
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version:
|
124
|
+
version: '0'
|
125
125
|
requirements: []
|
126
126
|
rubygems_version: 3.2.22
|
127
127
|
signing_key:
|