fluent-plugin-cmetrics 0.1.0.rc5 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24fd1261db862a55b33f49932214689ec9b65bdcd8aa0c3287993f21bba26943
|
4
|
+
data.tar.gz: c264c77b3be6894a9a36a5053b142ed8b27fb3fc89c6b2de74b36bc6ec407d01
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0db025054c9c808875e29612dba17abccdb6fc6f3337cd41f2b68a2c6429b3133eb07cd8a4fb3e0a38f532a70f0c241bc597675f1767057db6fa8ca40f3d548
|
7
|
+
data.tar.gz: fa1764de2e06cc32c2d880366b98d3912224500e7214cd3cb75231813c09515b865279ec7e9a0ea639638968e1c624df2efec7c5706678c0f175a8719d63595e
|
data/README.md
CHANGED
@@ -52,6 +52,26 @@ format to Splunk metrics
|
|
52
52
|
|
53
53
|
dimensions key
|
54
54
|
|
55
|
+
|
56
|
+
### \<fields\> section (optional) (single)
|
57
|
+
|
58
|
+
This secsion is used for adding extra fields into cmetrics msgpack payload parsed records.
|
59
|
+
|
60
|
+
For example, the following configuration should add hostname records into parsed records:
|
61
|
+
|
62
|
+
```aconf
|
63
|
+
<filter super.awesome.tag.**>
|
64
|
+
@type cmetrics_parser
|
65
|
+
format_to_splunk_metric true
|
66
|
+
dimensions_key dims
|
67
|
+
<fields>
|
68
|
+
hostname
|
69
|
+
</fields>
|
70
|
+
</filter>
|
71
|
+
```
|
72
|
+
|
73
|
+
On later data pipeline, `hostname` key can be used as some additional work.
|
74
|
+
|
55
75
|
## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
|
56
76
|
|
57
77
|
### cmetrics_name_key (string) (optional)
|
@@ -90,6 +110,16 @@ Specify splunk source name
|
|
90
110
|
|
91
111
|
Specify splunk sourcetype name
|
92
112
|
|
113
|
+
### only_use_last_field_keys (bool) (optional)
|
114
|
+
|
115
|
+
Whether using only last key values or not when processing for field keys
|
116
|
+
|
117
|
+
Default value: `false`
|
118
|
+
|
119
|
+
### \<fields\> section (optional) (single)
|
120
|
+
|
121
|
+
This secsion is used for adding extra fields as dimensions into cmetrics msgpack Splunk metric payload.
|
122
|
+
|
93
123
|
## Copyright
|
94
124
|
|
95
125
|
* Copyright(c) 2021- Calyptia Inc.
|
@@ -34,12 +34,23 @@ module Fluent
|
|
34
34
|
config_param :format_to_splunk_metric, :bool, default: false
|
35
35
|
desc "dimensions key"
|
36
36
|
config_param :dimensions_key, :string, default: nil
|
37
|
+
desc "Add additional records for Splunk dimensions"
|
38
|
+
config_section :fields, init: false, multi: false,required: false do
|
39
|
+
# Nothing here. For later purpose.
|
40
|
+
end
|
37
41
|
|
38
42
|
def configure(conf)
|
39
43
|
super
|
40
44
|
@serde = ::CMetrics::Serde.new
|
41
45
|
@record_accessor = record_accessor_create(@cmetrics_metric_key)
|
42
46
|
@labels_accessor = record_accessor_create(@cmetrics_labels_key)
|
47
|
+
@fields_accessors = {}
|
48
|
+
conf.elements(name: "fields").each do |e|
|
49
|
+
e.each_pair{|k, _v|
|
50
|
+
e.has_key?(k) # Suppress unused warnings.
|
51
|
+
@fields_accessors[k] = record_accessor_create(k)
|
52
|
+
}
|
53
|
+
end
|
43
54
|
end
|
44
55
|
|
45
56
|
def format_to_splunk_style_with_dims(inner)
|
@@ -59,6 +70,10 @@ module Fluent
|
|
59
70
|
new_es = Fluent::MultiEventStream.new
|
60
71
|
es.each do |time, record|
|
61
72
|
data = @record_accessor.call(record)
|
73
|
+
extra_fields = {}
|
74
|
+
@fields_accessors.each do |key, accessor|
|
75
|
+
extra_fields[key] = accessor.call(record)
|
76
|
+
end
|
62
77
|
@serde.feed_each(data) do |cmetrics|
|
63
78
|
metrics = cmetrics.metrics
|
64
79
|
metrics.each do |metric|
|
@@ -73,6 +88,9 @@ module Fluent
|
|
73
88
|
inner.merge!(dims)
|
74
89
|
end
|
75
90
|
end
|
91
|
+
if @fields_accessors
|
92
|
+
inner.merge!(extra_fields)
|
93
|
+
end
|
76
94
|
time = Time.at(inner.delete("timestamp"))
|
77
95
|
new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)
|
78
96
|
end
|
@@ -44,6 +44,11 @@ module Fluent
|
|
44
44
|
config_param :source, :string, default: nil
|
45
45
|
desc "Specify splunk sourcetype name"
|
46
46
|
config_param :sourcetype, :string, default: nil
|
47
|
+
desc "Whether using only last key values or not when processing for field keys"
|
48
|
+
config_param :only_use_last_field_keys, :bool, default: false
|
49
|
+
config_section :fields, init: false, multi: false, required: false do
|
50
|
+
# Other parameters nothing here. For later purpose.
|
51
|
+
end
|
47
52
|
|
48
53
|
def initialize
|
49
54
|
super
|
@@ -57,6 +62,13 @@ module Fluent
|
|
57
62
|
@cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
|
58
63
|
@cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
|
59
64
|
@host_key_accessor = record_accessor_create(@host_key)
|
65
|
+
@fields_accessors = {}
|
66
|
+
conf.elements(name: "fields").each do |e|
|
67
|
+
e.each_pair{|k, _v|
|
68
|
+
e.has_key?(k) # Suppress unused warnings.
|
69
|
+
@fields_accessors[k] = record_accessor_create(k)
|
70
|
+
}
|
71
|
+
end
|
60
72
|
end
|
61
73
|
|
62
74
|
def format(tag, time, record)
|
@@ -65,6 +77,8 @@ module Fluent
|
|
65
77
|
else
|
66
78
|
@default_host
|
67
79
|
end
|
80
|
+
extra_fields = {}
|
81
|
+
|
68
82
|
payload = {
|
69
83
|
host: host,
|
70
84
|
# From the API reference
|
@@ -81,6 +95,18 @@ module Fluent
|
|
81
95
|
if dims = @cmetrics_dims_accessor.call(record)
|
82
96
|
fields.merge!(dims)
|
83
97
|
end
|
98
|
+
@fields_accessors.each do |key, accessor|
|
99
|
+
if record = accessor.call(record)
|
100
|
+
record.each do |k, v|
|
101
|
+
if @only_use_last_field_keys
|
102
|
+
extra_fields[k.split(".").last] = v
|
103
|
+
else
|
104
|
+
extra_fields[k] = v
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
fields.merge!(extra_fields)
|
109
|
+
end
|
84
110
|
payload.merge!(fields)
|
85
111
|
Yajl.dump(payload)
|
86
112
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require "helper"
|
2
2
|
require "fluent/plugin/filter_cmetrics_parser.rb"
|
3
|
+
require 'socket'
|
3
4
|
|
4
5
|
class CmetricsParserTest < Test::Unit::TestCase
|
5
6
|
setup do
|
@@ -49,6 +50,37 @@ class CmetricsParserTest < Test::Unit::TestCase
|
|
49
50
|
d.feed(time, record)
|
50
51
|
end
|
51
52
|
d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
|
53
|
+
d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
|
54
|
+
assert do
|
55
|
+
d.filtered.size > 0
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
data("with dimensions" => "dims",
|
60
|
+
"without dimensions" => nil)
|
61
|
+
test "#filter_stream with host_key" do |data|
|
62
|
+
use_dimensions = data
|
63
|
+
d = if use_dimensions
|
64
|
+
create_driver(Fluent::Config::Element.new('ROOT', '', {
|
65
|
+
"format_to_splunk_metric" => true,
|
66
|
+
"dimensions_key" => "dims",
|
67
|
+
}, [
|
68
|
+
Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
|
69
|
+
]))
|
70
|
+
else
|
71
|
+
create_driver(Fluent::Config::Element.new('ROOT', '', {
|
72
|
+
"format_to_splunk_metric" => true,
|
73
|
+
}, [
|
74
|
+
Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
|
75
|
+
]))
|
76
|
+
end
|
77
|
+
time = event_time("2012-01-02 13:14:15")
|
78
|
+
record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
|
79
|
+
d.run(default_tag: 'test') do
|
80
|
+
d.feed(time, record)
|
81
|
+
end
|
82
|
+
d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
|
83
|
+
d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
|
52
84
|
assert do
|
53
85
|
d.filtered.size > 0
|
54
86
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cmetrics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroshi Hatake
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-12-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -119,9 +119,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
119
119
|
version: '0'
|
120
120
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- - "
|
122
|
+
- - ">="
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version:
|
124
|
+
version: '0'
|
125
125
|
requirements: []
|
126
126
|
rubygems_version: 3.2.22
|
127
127
|
signing_key:
|