fluent-plugin-cmetrics 0.1.0.rc4 → 0.1.0.rc8
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e207c02b8b1d7751c344573ec09c0d1ccf56c83d99776b2dade2e3b1366cb3d7
|
4
|
+
data.tar.gz: 448a06563a66ff12155a5059598c6fb99334f178f4564c0220c7baaecbc19eed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7219f2904e99a3b8f48f6fbaaa89d4261169563919be1d02552322b78b2756e56218e1982ef36bb93932813659e7c20ef40724f2768d6c2568f0b2fab24fc7f
|
7
|
+
data.tar.gz: fee515f5d3ae1cf09ce5af4fe4e45a6a471255d9db3ab4e9226c53cf74ea5146e764956fba8e78d16ddb65fee5e6a8b48530213d9c1d0e075cd54864d5fc4ec5
|
data/README.md
CHANGED
@@ -32,22 +32,87 @@ $ bundle
|
|
32
32
|
|
33
33
|
## Fluent::Plugin::CMetricsParserFilter
|
34
34
|
|
35
|
-
###
|
35
|
+
### cmetrics_metric_key (string) (optional)
|
36
36
|
|
37
37
|
cmetrics metric key
|
38
38
|
|
39
39
|
Default value: `cmetrics`.
|
40
40
|
|
41
|
-
###
|
41
|
+
### cmetrics_labels_key (string) (optional)
|
42
42
|
|
43
43
|
cmetrics labels key
|
44
44
|
|
45
45
|
Default value: `labels`.
|
46
46
|
|
47
|
-
###
|
47
|
+
### format_to_splunk_metric (bool) (optional)
|
48
48
|
|
49
|
-
format
|
49
|
+
format to Splunk metrics
|
50
50
|
|
51
|
+
### dimensions_key (string) (optional)
|
52
|
+
|
53
|
+
dimensions key
|
54
|
+
|
55
|
+
|
56
|
+
### \<fields\> section (optional) (single)
|
57
|
+
|
58
|
+
This secsion is used for adding extra fields into cmetrics msgpack payload parsed records.
|
59
|
+
|
60
|
+
For example, the following configuration should add hostname records into parsed records:
|
61
|
+
|
62
|
+
```aconf
|
63
|
+
<filter super.awesome.tag.**>
|
64
|
+
@type cmetrics_parser
|
65
|
+
format_to_splunk_metric true
|
66
|
+
dimensions_key dims
|
67
|
+
<fields>
|
68
|
+
hostname
|
69
|
+
</fields>
|
70
|
+
</filter>
|
71
|
+
```
|
72
|
+
|
73
|
+
On later data pipeline, `hostname` key can be used as some additional work.
|
74
|
+
|
75
|
+
## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
|
76
|
+
|
77
|
+
### cmetrics_name_key (string) (optional)
|
78
|
+
|
79
|
+
cmetrics metrics name key
|
80
|
+
|
81
|
+
Default value: `name`.
|
82
|
+
|
83
|
+
### cmetrics_value_key (string) (optional)
|
84
|
+
|
85
|
+
cmetrics metrics value key
|
86
|
+
|
87
|
+
Default value: `value`.
|
88
|
+
|
89
|
+
### cmetrics_dims_key (string) (optional)
|
90
|
+
|
91
|
+
cmetrics metrics dimensions key
|
92
|
+
|
93
|
+
Default value: `dims`.
|
94
|
+
|
95
|
+
### host_key (string) (optional)
|
96
|
+
|
97
|
+
Specify host key
|
98
|
+
|
99
|
+
Default value: `host`.
|
100
|
+
|
101
|
+
### index (string) (optional)
|
102
|
+
|
103
|
+
Specify splunk index name
|
104
|
+
|
105
|
+
### source (string) (optional)
|
106
|
+
|
107
|
+
Specify splunk source name
|
108
|
+
|
109
|
+
### sourcetype (string) (optional)
|
110
|
+
|
111
|
+
Specify splunk sourcetype name
|
112
|
+
|
113
|
+
### \<fields\> section (optional) (single)
|
114
|
+
|
115
|
+
This secsion is used for adding extra fields as dimensions into cmetrics msgpack Splunk metric payload.
|
51
116
|
|
52
117
|
## Copyright
|
53
118
|
|
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-cmetrics"
|
6
|
-
spec.version = "0.1.0.
|
6
|
+
spec.version = "0.1.0.rc8"
|
7
7
|
spec.authors = ["Hiroshi Hatake"]
|
8
8
|
spec.email = ["cosmo0920.oucc@gmail.com"]
|
9
9
|
|
@@ -30,43 +30,66 @@ module Fluent
|
|
30
30
|
config_param :cmetrics_metric_key, :string, default: "cmetrics"
|
31
31
|
desc "cmetrics labels key"
|
32
32
|
config_param :cmetrics_labels_key, :string, default: "labels"
|
33
|
-
desc "format
|
34
|
-
config_param :
|
33
|
+
desc "format to Splunk metrics"
|
34
|
+
config_param :format_to_splunk_metric, :bool, default: false
|
35
|
+
desc "dimensions key"
|
36
|
+
config_param :dimensions_key, :string, default: nil
|
37
|
+
desc "Add additional records for Splunk dimensions"
|
38
|
+
config_section :fields, init: false, multi: false,required: false do
|
39
|
+
# Nothing here. For later purpose.
|
40
|
+
end
|
35
41
|
|
36
42
|
def configure(conf)
|
37
43
|
super
|
38
44
|
@serde = ::CMetrics::Serde.new
|
39
45
|
@record_accessor = record_accessor_create(@cmetrics_metric_key)
|
40
46
|
@labels_accessor = record_accessor_create(@cmetrics_labels_key)
|
47
|
+
@fields_accessors = {}
|
48
|
+
conf.elements(name: "fields").each do |e|
|
49
|
+
e.each_pair{|k, _v|
|
50
|
+
e.has_key?(k) # Suppress unused warnings.
|
51
|
+
@fields_accessors[k] = record_accessor_create(k)
|
52
|
+
}
|
53
|
+
end
|
41
54
|
end
|
42
55
|
|
43
|
-
def
|
56
|
+
def format_to_splunk_style_with_dims(inner)
|
44
57
|
subsystem = inner.delete("subsystem")
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
}.join(".")
|
53
|
-
end
|
58
|
+
# labels will be treated as dimensions.
|
59
|
+
dimensions = Hash.new(0)
|
60
|
+
if labels = @labels_accessor.call(inner)
|
61
|
+
labels.map {|k,v|
|
62
|
+
dimensions[k] = v
|
63
|
+
}
|
64
|
+
end
|
54
65
|
name = inner.delete("name")
|
55
|
-
[subsystem,
|
66
|
+
return [subsystem, name].compact.reject{|e| e.empty?}.join("."), dimensions
|
56
67
|
end
|
57
68
|
|
58
69
|
def filter_stream(tag, es)
|
59
70
|
new_es = Fluent::MultiEventStream.new
|
60
71
|
es.each do |time, record|
|
61
72
|
data = @record_accessor.call(record)
|
73
|
+
extra_fields = {}
|
74
|
+
@fields_accessors.each do |key, accessor|
|
75
|
+
extra_fields[key] = accessor.call(record)
|
76
|
+
end
|
62
77
|
@serde.feed_each(data) do |cmetrics|
|
63
78
|
metrics = cmetrics.metrics
|
64
79
|
metrics.each do |metric|
|
65
80
|
next if metric.empty?
|
66
81
|
|
67
82
|
metric.each do |inner|
|
68
|
-
if @
|
69
|
-
inner["name"] =
|
83
|
+
if @format_to_splunk_metric
|
84
|
+
inner["name"], dims = format_to_splunk_style_with_dims(inner)
|
85
|
+
if @dimensions_key
|
86
|
+
inner[@dimensions_key] = dims
|
87
|
+
else
|
88
|
+
inner.merge!(dims)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
if @fields_accessors
|
92
|
+
inner.merge!(extra_fields)
|
70
93
|
end
|
71
94
|
time = Time.at(inner.delete("timestamp"))
|
72
95
|
new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)
|
@@ -34,6 +34,8 @@ module Fluent
|
|
34
34
|
config_param :cmetrics_name_key, :string, default: "name"
|
35
35
|
desc "cmetrics metrics value key"
|
36
36
|
config_param :cmetrics_value_key, :string, default: "value"
|
37
|
+
desc "cmetrics metrics dimensions key"
|
38
|
+
config_param :cmetrics_dims_key, :string, default: "dims"
|
37
39
|
desc "Specify host key"
|
38
40
|
config_param :host_key, :string, default: "host"
|
39
41
|
desc "Specify splunk index name"
|
@@ -42,6 +44,9 @@ module Fluent
|
|
42
44
|
config_param :source, :string, default: nil
|
43
45
|
desc "Specify splunk sourcetype name"
|
44
46
|
config_param :sourcetype, :string, default: nil
|
47
|
+
config_section :fields, init: false, multi: false,required: false do
|
48
|
+
# Nothing here. For later purpose.
|
49
|
+
end
|
45
50
|
|
46
51
|
def initialize
|
47
52
|
super
|
@@ -53,7 +58,15 @@ module Fluent
|
|
53
58
|
|
54
59
|
@cmetrics_name_accessor = record_accessor_create(@cmetrics_name_key)
|
55
60
|
@cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
|
61
|
+
@cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
|
56
62
|
@host_key_accessor = record_accessor_create(@host_key)
|
63
|
+
@fields_accessors = {}
|
64
|
+
conf.elements(name: "fields").each do |e|
|
65
|
+
e.each_pair{|k, _v|
|
66
|
+
e.has_key?(k) # Suppress unused warnings.
|
67
|
+
@fields_accessors[k] = record_accessor_create(k)
|
68
|
+
}
|
69
|
+
end
|
57
70
|
end
|
58
71
|
|
59
72
|
def format(tag, time, record)
|
@@ -62,6 +75,11 @@ module Fluent
|
|
62
75
|
else
|
63
76
|
@default_host
|
64
77
|
end
|
78
|
+
extra_fields = {}
|
79
|
+
|
80
|
+
@fields_accessors.each do |key, accessor|
|
81
|
+
extra_fields[key] = accessor.call(record)
|
82
|
+
end
|
65
83
|
payload = {
|
66
84
|
host: host,
|
67
85
|
# From the API reference
|
@@ -72,11 +90,16 @@ module Fluent
|
|
72
90
|
payload[:index] = @index if @index
|
73
91
|
payload[:source] = @source if @source
|
74
92
|
payload[:sourcetype] = @sourcetype if @sourcetype
|
75
|
-
|
76
|
-
metric_fields = {
|
93
|
+
fields = {
|
77
94
|
"metric_name:#{@cmetrics_name_accessor.call(record)}" => @cmetrics_value_accessor.call(record)
|
78
95
|
}
|
79
|
-
|
96
|
+
if dims = @cmetrics_dims_accessor.call(record)
|
97
|
+
fields.merge!(dims)
|
98
|
+
end
|
99
|
+
if @fields_accessors
|
100
|
+
fields.merge!(extra_fields)
|
101
|
+
end
|
102
|
+
payload.merge!(fields)
|
80
103
|
Yajl.dump(payload)
|
81
104
|
end
|
82
105
|
end
|
@@ -1,21 +1,27 @@
|
|
1
1
|
require "helper"
|
2
2
|
require "fluent/plugin/filter_cmetrics_parser.rb"
|
3
|
+
require 'socket'
|
3
4
|
|
4
5
|
class CmetricsParserTest < Test::Unit::TestCase
|
5
6
|
setup do
|
6
7
|
Fluent::Test.setup
|
7
8
|
end
|
8
9
|
|
9
|
-
data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.
|
10
|
-
"filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum"],
|
11
|
-
"disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.device
|
12
|
-
"network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.device
|
13
|
-
"none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5"])
|
14
|
-
test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key)|
|
10
|
+
data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.seconds_total", {"cpu" => "10", "mode" => "system"}],
|
11
|
+
"filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum", {}],
|
12
|
+
"disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.io_now", {"device" => "nvme0n1"}],
|
13
|
+
"network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.transmit_bytes_total", {"device" => "eth0"}],
|
14
|
+
"none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5", {}])
|
15
|
+
test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key, expected_dims)|
|
15
16
|
json = Yajl.load(json_str)
|
16
|
-
d = create_driver(%[
|
17
|
-
|
18
|
-
|
17
|
+
d = create_driver(%[
|
18
|
+
format_to_splunk_metric true
|
19
|
+
dimensions_key dims
|
20
|
+
])
|
21
|
+
assert_true d.instance.format_to_splunk_metric
|
22
|
+
formatted_key, dims = d.instance.format_to_splunk_style_with_dims(json)
|
23
|
+
assert_equal expected_format_key, formatted_key
|
24
|
+
assert_equal expected_dims, dims
|
19
25
|
end
|
20
26
|
|
21
27
|
sub_test_case "Actual filtering" do
|
@@ -24,13 +30,57 @@ class CmetricsParserTest < Test::Unit::TestCase
|
|
24
30
|
@binary = File.read(@binary_path)
|
25
31
|
end
|
26
32
|
|
27
|
-
|
28
|
-
|
33
|
+
data("with dimensions" => "dims",
|
34
|
+
"without dimensions" => nil)
|
35
|
+
test "#filter_stream" do |data|
|
36
|
+
use_dimensions = data
|
37
|
+
d = if use_dimensions
|
38
|
+
create_driver(%[
|
39
|
+
format_to_splunk_metric true
|
40
|
+
dimensions_key dims
|
41
|
+
])
|
42
|
+
else
|
43
|
+
create_driver(%[
|
44
|
+
format_to_splunk_metric true
|
45
|
+
])
|
46
|
+
end
|
29
47
|
time = event_time("2012-01-02 13:14:15")
|
30
48
|
record = {"cmetrics" => @binary}
|
31
49
|
d.run(default_tag: 'test') do
|
32
50
|
d.feed(time, record)
|
33
51
|
end
|
52
|
+
d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
|
53
|
+
d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
|
54
|
+
assert do
|
55
|
+
d.filtered.size > 0
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
data("with dimensions" => "dims",
|
60
|
+
"without dimensions" => nil)
|
61
|
+
test "#filter_stream with host_key" do |data|
|
62
|
+
use_dimensions = data
|
63
|
+
d = if use_dimensions
|
64
|
+
create_driver(Fluent::Config::Element.new('ROOT', '', {
|
65
|
+
"format_to_splunk_metric" => true,
|
66
|
+
"dimensions_key" => "dims",
|
67
|
+
}, [
|
68
|
+
Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
|
69
|
+
]))
|
70
|
+
else
|
71
|
+
create_driver(Fluent::Config::Element.new('ROOT', '', {
|
72
|
+
"format_to_splunk_metric" => true,
|
73
|
+
}, [
|
74
|
+
Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
|
75
|
+
]))
|
76
|
+
end
|
77
|
+
time = event_time("2012-01-02 13:14:15")
|
78
|
+
record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
|
79
|
+
d.run(default_tag: 'test') do
|
80
|
+
d.feed(time, record)
|
81
|
+
end
|
82
|
+
d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
|
83
|
+
d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
|
34
84
|
assert do
|
35
85
|
d.filtered.size > 0
|
36
86
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cmetrics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0.
|
4
|
+
version: 0.1.0.rc8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroshi Hatake
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-10-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|