fluent-plugin-cmetrics 0.1.0.rc3 → 0.1.0.rc7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f2da1d6e9e9b821b5992d1e6989d034d04f074bc10a11ecd35d2842207a77754
|
|
4
|
+
data.tar.gz: c71b5a5c8ad4a33328b4c542d00a2a562639fa9f9b646db3bd9c66cf922c2b53
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5cd27f38bb9ea19e325589293f29cf13b07bf3158adb0d21febc502d6f5c2e76ed8841ebea47436478890c79fa9ac546918abd91bce17f48ef2970a71695bc38
|
|
7
|
+
data.tar.gz: 6661de3897ecac7c2a07a5765dbf16f9c6876e6ab2d6ae80d789793473c67ca91ac3fe4a8091335f99dcc517c4343253e3369c5cdd2c977d3d4141a85b82fd19
|
data/README.md
CHANGED
|
@@ -32,22 +32,83 @@ $ bundle
|
|
|
32
32
|
|
|
33
33
|
## Fluent::Plugin::CMetricsParserFilter
|
|
34
34
|
|
|
35
|
-
###
|
|
35
|
+
### cmetrics_metric_key (string) (optional)
|
|
36
36
|
|
|
37
37
|
cmetrics metric key
|
|
38
38
|
|
|
39
39
|
Default value: `cmetrics`.
|
|
40
40
|
|
|
41
|
-
###
|
|
41
|
+
### cmetrics_labels_key (string) (optional)
|
|
42
42
|
|
|
43
43
|
cmetrics labels key
|
|
44
44
|
|
|
45
45
|
Default value: `labels`.
|
|
46
46
|
|
|
47
|
-
###
|
|
47
|
+
### format_to_splunk_metric (bool) (optional)
|
|
48
48
|
|
|
49
|
-
format
|
|
49
|
+
format to Splunk metrics
|
|
50
50
|
|
|
51
|
+
### dimensions_key (string) (optional)
|
|
52
|
+
|
|
53
|
+
dimensions key
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
### \<fields\> section (optional) (single)
|
|
57
|
+
|
|
58
|
+
This secsion is used for adding extra fields into cmetrics msgpack payload parsed records.
|
|
59
|
+
|
|
60
|
+
For example, the following configuration should add hostname records into parsed records:
|
|
61
|
+
|
|
62
|
+
```aconf
|
|
63
|
+
<filter super.awesome.tag.**>
|
|
64
|
+
@type cmetrics_parser
|
|
65
|
+
format_to_splunk_metric true
|
|
66
|
+
dimensions_key dims
|
|
67
|
+
<fields>
|
|
68
|
+
hostname
|
|
69
|
+
</fields>
|
|
70
|
+
</filter>
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
On later data pipeline, `hostname` key can be used as some additional work.
|
|
74
|
+
|
|
75
|
+
## Fluent::Plugin::CMetricsSplunkMetricPayloadFormatter
|
|
76
|
+
|
|
77
|
+
### cmetrics_name_key (string) (optional)
|
|
78
|
+
|
|
79
|
+
cmetrics metrics name key
|
|
80
|
+
|
|
81
|
+
Default value: `name`.
|
|
82
|
+
|
|
83
|
+
### cmetrics_value_key (string) (optional)
|
|
84
|
+
|
|
85
|
+
cmetrics metrics value key
|
|
86
|
+
|
|
87
|
+
Default value: `value`.
|
|
88
|
+
|
|
89
|
+
### cmetrics_dims_key (string) (optional)
|
|
90
|
+
|
|
91
|
+
cmetrics metrics dimensions key
|
|
92
|
+
|
|
93
|
+
Default value: `dims`.
|
|
94
|
+
|
|
95
|
+
### host_key (string) (optional)
|
|
96
|
+
|
|
97
|
+
Specify host key
|
|
98
|
+
|
|
99
|
+
Default value: `host`.
|
|
100
|
+
|
|
101
|
+
### index (string) (optional)
|
|
102
|
+
|
|
103
|
+
Specify splunk index name
|
|
104
|
+
|
|
105
|
+
### source (string) (optional)
|
|
106
|
+
|
|
107
|
+
Specify splunk source name
|
|
108
|
+
|
|
109
|
+
### sourcetype (string) (optional)
|
|
110
|
+
|
|
111
|
+
Specify splunk sourcetype name
|
|
51
112
|
|
|
52
113
|
## Copyright
|
|
53
114
|
|
|
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |spec|
|
|
5
5
|
spec.name = "fluent-plugin-cmetrics"
|
|
6
|
-
spec.version = "0.1.0.
|
|
6
|
+
spec.version = "0.1.0.rc7"
|
|
7
7
|
spec.authors = ["Hiroshi Hatake"]
|
|
8
8
|
spec.email = ["cosmo0920.oucc@gmail.com"]
|
|
9
9
|
|
|
@@ -30,43 +30,66 @@ module Fluent
|
|
|
30
30
|
config_param :cmetrics_metric_key, :string, default: "cmetrics"
|
|
31
31
|
desc "cmetrics labels key"
|
|
32
32
|
config_param :cmetrics_labels_key, :string, default: "labels"
|
|
33
|
-
desc "format
|
|
34
|
-
config_param :
|
|
33
|
+
desc "format to Splunk metrics"
|
|
34
|
+
config_param :format_to_splunk_metric, :bool, default: false
|
|
35
|
+
desc "dimensions key"
|
|
36
|
+
config_param :dimensions_key, :string, default: nil
|
|
37
|
+
desc "Add additional records for Splunk dimensions"
|
|
38
|
+
config_section :fields, init: false, multi: false,required: false do
|
|
39
|
+
# Nothing here. For later purpose.
|
|
40
|
+
end
|
|
35
41
|
|
|
36
42
|
def configure(conf)
|
|
37
43
|
super
|
|
38
44
|
@serde = ::CMetrics::Serde.new
|
|
39
45
|
@record_accessor = record_accessor_create(@cmetrics_metric_key)
|
|
40
46
|
@labels_accessor = record_accessor_create(@cmetrics_labels_key)
|
|
47
|
+
@fields_accessors = {}
|
|
48
|
+
conf.elements(name: "fields").each do |e|
|
|
49
|
+
e.each_pair{|k, _v|
|
|
50
|
+
e.has_key?(k) # Suppress unused warnings.
|
|
51
|
+
@fields_accessors[k] = record_accessor_create(k)
|
|
52
|
+
}
|
|
53
|
+
end
|
|
41
54
|
end
|
|
42
55
|
|
|
43
|
-
def
|
|
56
|
+
def format_to_splunk_style_with_dims(inner)
|
|
44
57
|
subsystem = inner.delete("subsystem")
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
}.join(".")
|
|
53
|
-
end
|
|
58
|
+
# labels will be treated as dimensions.
|
|
59
|
+
dimensions = Hash.new(0)
|
|
60
|
+
if labels = @labels_accessor.call(inner)
|
|
61
|
+
labels.map {|k,v|
|
|
62
|
+
dimensions[k] = v
|
|
63
|
+
}
|
|
64
|
+
end
|
|
54
65
|
name = inner.delete("name")
|
|
55
|
-
[subsystem,
|
|
66
|
+
return [subsystem, name].compact.reject{|e| e.empty?}.join("."), dimensions
|
|
56
67
|
end
|
|
57
68
|
|
|
58
69
|
def filter_stream(tag, es)
|
|
59
70
|
new_es = Fluent::MultiEventStream.new
|
|
60
71
|
es.each do |time, record|
|
|
61
72
|
data = @record_accessor.call(record)
|
|
73
|
+
extra_fields = {}
|
|
74
|
+
@fields_accessors.each do |key, accessor|
|
|
75
|
+
extra_fields[key] = accessor.call(record)
|
|
76
|
+
end
|
|
62
77
|
@serde.feed_each(data) do |cmetrics|
|
|
63
78
|
metrics = cmetrics.metrics
|
|
64
79
|
metrics.each do |metric|
|
|
65
80
|
next if metric.empty?
|
|
66
81
|
|
|
67
82
|
metric.each do |inner|
|
|
68
|
-
if @
|
|
69
|
-
inner["name"] =
|
|
83
|
+
if @format_to_splunk_metric
|
|
84
|
+
inner["name"], dims = format_to_splunk_style_with_dims(inner)
|
|
85
|
+
if @dimensions_key
|
|
86
|
+
inner[@dimensions_key] = dims
|
|
87
|
+
else
|
|
88
|
+
inner.merge!(dims)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
if @fields_accessors
|
|
92
|
+
inner.merge!(extra_fields)
|
|
70
93
|
end
|
|
71
94
|
time = Time.at(inner.delete("timestamp"))
|
|
72
95
|
new_es.add(Fluent::EventTime.new(time.to_i, time.nsec), inner)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright 2021- Calyptia Inc.
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
require "fluent/plugin/formatter"
|
|
17
|
+
require 'fluent/plugin_helper/record_accessor'
|
|
18
|
+
require 'fluent/plugin_helper'
|
|
19
|
+
require "fluent/event"
|
|
20
|
+
require "fluent/time"
|
|
21
|
+
require "yajl"
|
|
22
|
+
require "socket"
|
|
23
|
+
|
|
24
|
+
module Fluent
|
|
25
|
+
module Plugin
|
|
26
|
+
class CMetricsSplunkMetricPayloadFormatter < Fluent::Plugin::Formatter
|
|
27
|
+
include PluginHelper::Mixin
|
|
28
|
+
|
|
29
|
+
Fluent::Plugin.register_formatter('cmetrics_splunk_metric_payload', self)
|
|
30
|
+
|
|
31
|
+
helpers :record_accessor
|
|
32
|
+
|
|
33
|
+
desc "cmetrics metrics name key"
|
|
34
|
+
config_param :cmetrics_name_key, :string, default: "name"
|
|
35
|
+
desc "cmetrics metrics value key"
|
|
36
|
+
config_param :cmetrics_value_key, :string, default: "value"
|
|
37
|
+
desc "cmetrics metrics dimensions key"
|
|
38
|
+
config_param :cmetrics_dims_key, :string, default: "dims"
|
|
39
|
+
desc "Specify host key"
|
|
40
|
+
config_param :host_key, :string, default: "host"
|
|
41
|
+
desc "Specify splunk index name"
|
|
42
|
+
config_param :index, :string, default: nil
|
|
43
|
+
desc "Specify splunk source name"
|
|
44
|
+
config_param :source, :string, default: nil
|
|
45
|
+
desc "Specify splunk sourcetype name"
|
|
46
|
+
config_param :sourcetype, :string, default: nil
|
|
47
|
+
|
|
48
|
+
def initialize
|
|
49
|
+
super
|
|
50
|
+
@default_host = Socket.gethostname
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def configure(conf)
|
|
54
|
+
super
|
|
55
|
+
|
|
56
|
+
@cmetrics_name_accessor = record_accessor_create(@cmetrics_name_key)
|
|
57
|
+
@cmetrics_value_accessor = record_accessor_create(@cmetrics_value_key)
|
|
58
|
+
@cmetrics_dims_accessor = record_accessor_create(@cmetrics_dims_key)
|
|
59
|
+
@host_key_accessor = record_accessor_create(@host_key)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def format(tag, time, record)
|
|
63
|
+
host = if host = @host_key_accessor.call(record)
|
|
64
|
+
host
|
|
65
|
+
else
|
|
66
|
+
@default_host
|
|
67
|
+
end
|
|
68
|
+
payload = {
|
|
69
|
+
host: host,
|
|
70
|
+
# From the API reference
|
|
71
|
+
# https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
|
|
72
|
+
time: time.to_f.to_s,
|
|
73
|
+
event: 'metric',
|
|
74
|
+
}
|
|
75
|
+
payload[:index] = @index if @index
|
|
76
|
+
payload[:source] = @source if @source
|
|
77
|
+
payload[:sourcetype] = @sourcetype if @sourcetype
|
|
78
|
+
fields = {
|
|
79
|
+
"metric_name:#{@cmetrics_name_accessor.call(record)}" => @cmetrics_value_accessor.call(record)
|
|
80
|
+
}
|
|
81
|
+
if dims = @cmetrics_dims_accessor.call(record)
|
|
82
|
+
fields.merge!(dims)
|
|
83
|
+
end
|
|
84
|
+
payload.merge!(fields)
|
|
85
|
+
Yajl.dump(payload)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -1,21 +1,27 @@
|
|
|
1
1
|
require "helper"
|
|
2
2
|
require "fluent/plugin/filter_cmetrics_parser.rb"
|
|
3
|
+
require 'socket'
|
|
3
4
|
|
|
4
5
|
class CmetricsParserTest < Test::Unit::TestCase
|
|
5
6
|
setup do
|
|
6
7
|
Fluent::Test.setup
|
|
7
8
|
end
|
|
8
9
|
|
|
9
|
-
data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.
|
|
10
|
-
"filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum"],
|
|
11
|
-
"disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.device
|
|
12
|
-
"network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.device
|
|
13
|
-
"none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5"])
|
|
14
|
-
test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key)|
|
|
10
|
+
data("cpu" => ['{"namespace":"node","subsystem":"cpu","labels":{"cpu":"10","mode":"system"},"name":"seconds_total","description":"Seconds the CPUs spent in each mode.","value":13153.09}', "cpu.seconds_total", {"cpu" => "10", "mode" => "system"}],
|
|
11
|
+
"filefd" => ['{"namespace":"node","subsystem":"filefd","name":"maximum","description":"File descriptor statistics: maximum.","value":9.223372036854776e+18}', "filefd.maximum", {}],
|
|
12
|
+
"disk" => ['{"namespace":"node","subsystem":"disk","labels":{"device":"nvme0n1"},"name":"io_now","description":"The number of I/Os currently in progress.","value":0.0}', "disk.io_now", {"device" => "nvme0n1"}],
|
|
13
|
+
"network" => ['{"namespace":"node","subsystem":"network","labels":{"device":"eth0"},"name":"transmit_bytes_total","description":"Network device statistic bytes.","value":997193.0}', "network.transmit_bytes_total", {"device" => "eth0"}],
|
|
14
|
+
"none" => ['{"namespace":"node","subsystem":"","name":"load5","description":"5m load average.","value":0.94}', "load5", {}])
|
|
15
|
+
test "#format_record_key_to_splunk_style" do |(json_str, expected_format_key, expected_dims)|
|
|
15
16
|
json = Yajl.load(json_str)
|
|
16
|
-
d = create_driver(%[
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
d = create_driver(%[
|
|
18
|
+
format_to_splunk_metric true
|
|
19
|
+
dimensions_key dims
|
|
20
|
+
])
|
|
21
|
+
assert_true d.instance.format_to_splunk_metric
|
|
22
|
+
formatted_key, dims = d.instance.format_to_splunk_style_with_dims(json)
|
|
23
|
+
assert_equal expected_format_key, formatted_key
|
|
24
|
+
assert_equal expected_dims, dims
|
|
19
25
|
end
|
|
20
26
|
|
|
21
27
|
sub_test_case "Actual filtering" do
|
|
@@ -24,13 +30,57 @@ class CmetricsParserTest < Test::Unit::TestCase
|
|
|
24
30
|
@binary = File.read(@binary_path)
|
|
25
31
|
end
|
|
26
32
|
|
|
27
|
-
|
|
28
|
-
|
|
33
|
+
data("with dimensions" => "dims",
|
|
34
|
+
"without dimensions" => nil)
|
|
35
|
+
test "#filter_stream" do |data|
|
|
36
|
+
use_dimensions = data
|
|
37
|
+
d = if use_dimensions
|
|
38
|
+
create_driver(%[
|
|
39
|
+
format_to_splunk_metric true
|
|
40
|
+
dimensions_key dims
|
|
41
|
+
])
|
|
42
|
+
else
|
|
43
|
+
create_driver(%[
|
|
44
|
+
format_to_splunk_metric true
|
|
45
|
+
])
|
|
46
|
+
end
|
|
29
47
|
time = event_time("2012-01-02 13:14:15")
|
|
30
48
|
record = {"cmetrics" => @binary}
|
|
31
49
|
d.run(default_tag: 'test') do
|
|
32
50
|
d.feed(time, record)
|
|
33
51
|
end
|
|
52
|
+
d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
|
|
53
|
+
d.filtered.map {|e| assert_false(e.last.has_key?("hostname"))}
|
|
54
|
+
assert do
|
|
55
|
+
d.filtered.size > 0
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
data("with dimensions" => "dims",
|
|
60
|
+
"without dimensions" => nil)
|
|
61
|
+
test "#filter_stream with host_key" do |data|
|
|
62
|
+
use_dimensions = data
|
|
63
|
+
d = if use_dimensions
|
|
64
|
+
create_driver(Fluent::Config::Element.new('ROOT', '', {
|
|
65
|
+
"format_to_splunk_metric" => true,
|
|
66
|
+
"dimensions_key" => "dims",
|
|
67
|
+
}, [
|
|
68
|
+
Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
|
|
69
|
+
]))
|
|
70
|
+
else
|
|
71
|
+
create_driver(Fluent::Config::Element.new('ROOT', '', {
|
|
72
|
+
"format_to_splunk_metric" => true,
|
|
73
|
+
}, [
|
|
74
|
+
Fluent::Config::Element.new('fields', '', {"hostname" => ""}, [])
|
|
75
|
+
]))
|
|
76
|
+
end
|
|
77
|
+
time = event_time("2012-01-02 13:14:15")
|
|
78
|
+
record = {"cmetrics" => @binary, "hostname" => Socket.gethostname}
|
|
79
|
+
d.run(default_tag: 'test') do
|
|
80
|
+
d.feed(time, record)
|
|
81
|
+
end
|
|
82
|
+
d.filtered.map {|e| assert_equal(!!use_dimensions, e.last.has_key?("dims"))}
|
|
83
|
+
d.filtered.map {|e| assert_true(e.last.has_key?("hostname"))}
|
|
34
84
|
assert do
|
|
35
85
|
d.filtered.size > 0
|
|
36
86
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-cmetrics
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.0.
|
|
4
|
+
version: 0.1.0.rc7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Hiroshi Hatake
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-09-
|
|
11
|
+
date: 2021-09-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -100,6 +100,7 @@ files:
|
|
|
100
100
|
- Rakefile
|
|
101
101
|
- fluent-plugin-cmetrics.gemspec
|
|
102
102
|
- lib/fluent/plugin/filter_cmetrics_parser.rb
|
|
103
|
+
- lib/fluent/plugin/formatter_cmetrics_splunk_metric_payload.rb
|
|
103
104
|
- test/fixtures/cmetrics.bin
|
|
104
105
|
- test/helper.rb
|
|
105
106
|
- test/plugin/test_filter_cmetrics_parser.rb
|
|
@@ -122,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
122
123
|
- !ruby/object:Gem::Version
|
|
123
124
|
version: 1.3.1
|
|
124
125
|
requirements: []
|
|
125
|
-
rubygems_version: 3.
|
|
126
|
+
rubygems_version: 3.2.22
|
|
126
127
|
signing_key:
|
|
127
128
|
specification_version: 4
|
|
128
129
|
summary: Fluentd plugin for cmetrics format handling.
|