fluent-plugin-influxdb-v2 1.5.0.pre.579
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +157 -0
- data/.circleci/setup-rubygems.sh +3 -0
- data/.github/PULL_REQUEST_TEMPLATE +8 -0
- data/.gitignore +14 -0
- data/.rubocop.yml +36 -0
- data/CHANGELOG.md +38 -0
- data/Gemfile +24 -0
- data/LICENSE +21 -0
- data/README.md +113 -0
- data/Rakefile +37 -0
- data/bin/influxdb-onboarding.sh +39 -0
- data/bin/influxdb-restart.sh +60 -0
- data/examples/README.md +190 -0
- data/examples/ab/load.sh +25 -0
- data/examples/ab/urls.txt +4 -0
- data/examples/architecture.png +0 -0
- data/examples/dashboard.png +0 -0
- data/examples/fluentd/Dockerfile +12 -0
- data/examples/fluentd/entrypoint.sh +28 -0
- data/examples/fluentd/fluent.conf +40 -0
- data/examples/influxdb/web_app_access.json +491 -0
- data/examples/run-example.sh +113 -0
- data/examples/web/httpd.conf +555 -0
- data/fluent-plugin-influxdb-v2.gemspec +61 -0
- data/lib/fluent/plugin/fluent.rb +22 -0
- data/lib/fluent/plugin/out_influxdb2.rb +160 -0
- data/lib/fluent/plugin/version.rb +27 -0
- data/test/influxdb/plugin/output_test.rb +443 -0
- data/test/influxdb/plugin/version_test.rb +27 -0
- data/test/test_helper.rb +40 -0
- metadata +233 -0
@@ -0,0 +1,61 @@
|
|
1
|
+
# The MIT License
|
2
|
+
#
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
4
|
+
# of this software and associated documentation files (the "Software"), to deal
|
5
|
+
# in the Software without restriction, including without limitation the rights
|
6
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
7
|
+
# copies of the Software, and to permit persons to whom the Software is
|
8
|
+
# furnished to do so, subject to the following conditions:
|
9
|
+
#
|
10
|
+
# The above copyright notice and this permission notice shall be included in
|
11
|
+
# all copies or substantial portions of the Software.
|
12
|
+
#
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
19
|
+
# THE SOFTWARE.
|
20
|
+
#
|
21
|
+
lib = File.expand_path('lib', __dir__)
|
22
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
23
|
+
require 'fluent/plugin/version'
|
24
|
+
|
25
|
+
Gem::Specification.new do |spec|
|
26
|
+
spec.name = 'fluent-plugin-influxdb-v2'
|
27
|
+
spec.version = if ENV['CIRCLE_BUILD_NUM']
|
28
|
+
"#{InfluxDB2::Plugin::Fluent::VERSION}-#{ENV['CIRCLE_BUILD_NUM']}"
|
29
|
+
else
|
30
|
+
InfluxDB2::Plugin::Fluent::VERSION
|
31
|
+
end
|
32
|
+
spec.authors = ['Jakub Bednar']
|
33
|
+
spec.email = ['jakub.bednar@gmail.com']
|
34
|
+
|
35
|
+
spec.summary = 'InfluxDB 2 output plugin for Fluentd'
|
36
|
+
spec.description = 'A buffered output plugin for Fluentd and InfluxDB 2'
|
37
|
+
spec.homepage = 'https://github.com/influxdata/influxdb-plugin-fluent'
|
38
|
+
spec.license = 'MIT'
|
39
|
+
|
40
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
41
|
+
spec.metadata['source_code_uri'] = 'https://github.com/influxdata/influxdb-plugin-fluent'
|
42
|
+
spec.metadata['changelog_uri'] = 'https://raw.githubusercontent.com/influxdata/influxdb-plugin-fluent/master/CHANGELOG.md'
|
43
|
+
|
44
|
+
spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
|
45
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features|smoke)/})
|
46
|
+
spec.require_paths = ['lib']
|
47
|
+
spec.required_ruby_version = '>= 2.2.0'
|
48
|
+
|
49
|
+
spec.add_runtime_dependency 'fluentd', '~> 1.8'
|
50
|
+
spec.add_runtime_dependency 'influxdb-client', '1.6.0'
|
51
|
+
|
52
|
+
spec.add_development_dependency 'bundler', '~> 2.0'
|
53
|
+
spec.add_development_dependency 'codecov', '~> 0.1.16'
|
54
|
+
spec.add_development_dependency 'minitest', '~> 5.0'
|
55
|
+
spec.add_development_dependency 'minitest-reporters', '~> 1.4'
|
56
|
+
spec.add_development_dependency 'rake', '>= 12.3.3'
|
57
|
+
spec.add_development_dependency 'rubocop', '~> 0.66.0'
|
58
|
+
spec.add_development_dependency 'simplecov', '~> 0.17.1'
|
59
|
+
spec.add_development_dependency 'test-unit', '~> 3.3'
|
60
|
+
spec.add_development_dependency 'webmock', '~> 3.7'
|
61
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# The MIT License
|
2
|
+
#
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
4
|
+
# of this software and associated documentation files (the "Software"), to deal
|
5
|
+
# in the Software without restriction, including without limitation the rights
|
6
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
7
|
+
# copies of the Software, and to permit persons to whom the Software is
|
8
|
+
# furnished to do so, subject to the following conditions:
|
9
|
+
#
|
10
|
+
# The above copyright notice and this permission notice shall be included in
|
11
|
+
# all copies or substantial portions of the Software.
|
12
|
+
#
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
19
|
+
# THE SOFTWARE.
|
20
|
+
|
21
|
+
require 'fluent/plugin/version'
|
22
|
+
require 'fluent/plugin/out_influxdb2'
|
@@ -0,0 +1,160 @@
|
|
1
|
+
# The MIT License
|
2
|
+
#
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
4
|
+
# of this software and associated documentation files (the "Software"), to deal
|
5
|
+
# in the Software without restriction, including without limitation the rights
|
6
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
7
|
+
# copies of the Software, and to permit persons to whom the Software is
|
8
|
+
# furnished to do so, subject to the following conditions:
|
9
|
+
#
|
10
|
+
# The above copyright notice and this permission notice shall be included in
|
11
|
+
# all copies or substantial portions of the Software.
|
12
|
+
#
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
19
|
+
# THE SOFTWARE.
|
20
|
+
|
21
|
+
require 'fluent/plugin/output'
|
22
|
+
require 'influxdb-client'
|
23
|
+
|
24
|
+
# A buffered output plugin for Fluentd and InfluxDB 2
|
25
|
+
class InfluxDBOutput < Fluent::Plugin::Output
|
26
|
+
Fluent::Plugin.register_output('influxdb2', self)
|
27
|
+
|
28
|
+
helpers :inject, :compat_parameters
|
29
|
+
|
30
|
+
DEFAULT_BUFFER_TYPE = 'memory'.freeze
|
31
|
+
|
32
|
+
config_param :url, :string, default: 'https://localhost:9999'
|
33
|
+
desc 'InfluxDB URL to connect to (ex. https://localhost:9999).'
|
34
|
+
|
35
|
+
config_param :token, :string
|
36
|
+
desc 'Access Token used for authenticating/authorizing the InfluxDB request sent by client.'
|
37
|
+
|
38
|
+
config_param :use_ssl, :bool, default: true
|
39
|
+
desc 'Turn on/off SSL for HTTP communication.'
|
40
|
+
|
41
|
+
config_param :bucket, :string
|
42
|
+
desc 'Specifies the destination bucket for writes.'
|
43
|
+
|
44
|
+
config_param :org, :string
|
45
|
+
desc 'Specifies the destination organization for writes.'
|
46
|
+
|
47
|
+
config_param :measurement, :string, default: nil
|
48
|
+
desc 'The name of the measurement. If not specified, Fluentd\'s tag is used.'
|
49
|
+
|
50
|
+
config_param :tag_keys, :array, default: []
|
51
|
+
desc 'The list of record keys that are stored in InfluxDB as \'tag\'.'
|
52
|
+
|
53
|
+
config_param :tag_fluentd, :bool, default: false
|
54
|
+
desc 'Specifies if the Fluentd\'s event tag is included into InfluxDB tags (ex. \'fluentd=system.logs\').'
|
55
|
+
|
56
|
+
config_param :field_keys, :array, default: []
|
57
|
+
desc 'The list of record keys that are stored in InfluxDB as \'field\'. ' \
|
58
|
+
'If it\'s not specified than as fields are used all record keys.'
|
59
|
+
|
60
|
+
config_param :field_cast_to_float, :bool, default: false
|
61
|
+
desc 'Turn on/off auto casting Integer value to Float. ' \
|
62
|
+
'Helper to avoid mismatch error: \'series type mismatch: already Integer but got Float\'.'
|
63
|
+
|
64
|
+
config_param :time_precision, :string, default: 'ns'
|
65
|
+
desc 'The time precision of timestamp. You should specify either second (s), ' \
|
66
|
+
'millisecond (ms), microsecond (us), or nanosecond (ns).'
|
67
|
+
|
68
|
+
config_param :time_key, :string, default: nil
|
69
|
+
desc 'A name of the record key that used as a \'timestamp\' instead of event timestamp.' \
|
70
|
+
'If a record key doesn\'t exists or hasn\'t value then is used event timestamp.'
|
71
|
+
|
72
|
+
config_section :buffer do
|
73
|
+
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
74
|
+
config_set_default :chunk_keys, ['tag']
|
75
|
+
end
|
76
|
+
|
77
|
+
def configure(conf)
|
78
|
+
compat_parameters_convert(conf, :inject)
|
79
|
+
super
|
80
|
+
case @time_precision
|
81
|
+
when 'ns' then
|
82
|
+
@precision_formatter = ->(ns_time) { ns_time }
|
83
|
+
when 'us' then
|
84
|
+
@precision_formatter = ->(ns_time) { (ns_time / 1e3).round }
|
85
|
+
when 'ms' then
|
86
|
+
@precision_formatter = ->(ns_time) { (ns_time / 1e6).round }
|
87
|
+
when 's' then
|
88
|
+
@precision_formatter = ->(ns_time) { (ns_time / 1e9).round }
|
89
|
+
else
|
90
|
+
raise Fluent::ConfigError, "The time precision #{@time_precision} is not supported. You should use: " \
|
91
|
+
'second (s), millisecond (ms), microsecond (us), or nanosecond (ns).'
|
92
|
+
end
|
93
|
+
@precision = InfluxDB2::WritePrecision.new.get_from_value(@time_precision)
|
94
|
+
raise Fluent::ConfigError, 'The InfluxDB URL should be defined.' if @url.empty?
|
95
|
+
end
|
96
|
+
|
97
|
+
def start
|
98
|
+
super
|
99
|
+
log.info "Connecting to InfluxDB: url: #{@url}, bucket: #{@bucket}, org: #{@org}, precision = #{@precision}, " \
|
100
|
+
"use_ssl = #{@use_ssl}"
|
101
|
+
@client = InfluxDB2::Client.new(@url, @token, bucket: @bucket, org: @org, precision: @precision, use_ssl: @use_ssl)
|
102
|
+
@write_api = @client.create_write_api
|
103
|
+
end
|
104
|
+
|
105
|
+
def shutdown
|
106
|
+
super
|
107
|
+
@client.close!
|
108
|
+
end
|
109
|
+
|
110
|
+
def multi_workers_ready?
|
111
|
+
true
|
112
|
+
end
|
113
|
+
|
114
|
+
def write(chunk)
|
115
|
+
points = []
|
116
|
+
tag = chunk.metadata.tag
|
117
|
+
measurement = @measurement || tag
|
118
|
+
chunk.msgpack_each do |time, record|
|
119
|
+
if time.is_a?(Integer)
|
120
|
+
time_formatted = time
|
121
|
+
else
|
122
|
+
nano_seconds = time.sec * 1e9
|
123
|
+
nano_seconds += time.nsec
|
124
|
+
time_formatted = @precision_formatter.call(nano_seconds)
|
125
|
+
end
|
126
|
+
point = InfluxDB2::Point
|
127
|
+
.new(name: measurement)
|
128
|
+
record.each_pair do |k, v|
|
129
|
+
if k.eql?(@time_key)
|
130
|
+
time_formatted = v
|
131
|
+
else
|
132
|
+
_parse_field(k, v, point)
|
133
|
+
end
|
134
|
+
point.add_tag('fluentd', tag) if @tag_fluentd
|
135
|
+
end
|
136
|
+
point.time(time_formatted, @precision)
|
137
|
+
points << point
|
138
|
+
end
|
139
|
+
@write_api.write(data: points)
|
140
|
+
log.debug "Written points: #{points}"
|
141
|
+
end
|
142
|
+
|
143
|
+
private
|
144
|
+
|
145
|
+
def _parse_field(key, value, point)
|
146
|
+
if @tag_keys.include?(key)
|
147
|
+
point.add_tag(key, value)
|
148
|
+
elsif @field_keys.empty? || @field_keys.include?(key)
|
149
|
+
if @field_cast_to_float & value.is_a?(Integer)
|
150
|
+
point.add_field(key, Float(value))
|
151
|
+
elsif value.is_a?(Hash)
|
152
|
+
value.each_pair do |nested_k, nested_v|
|
153
|
+
_parse_field("#{key}.#{nested_k}", nested_v, point)
|
154
|
+
end
|
155
|
+
else
|
156
|
+
point.add_field(key, value)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# The MIT License
|
2
|
+
#
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
4
|
+
# of this software and associated documentation files (the "Software"), to deal
|
5
|
+
# in the Software without restriction, including without limitation the rights
|
6
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
7
|
+
# copies of the Software, and to permit persons to whom the Software is
|
8
|
+
# furnished to do so, subject to the following conditions:
|
9
|
+
#
|
10
|
+
# The above copyright notice and this permission notice shall be included in
|
11
|
+
# all copies or substantial portions of the Software.
|
12
|
+
#
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
19
|
+
# THE SOFTWARE.
|
20
|
+
|
21
|
+
module InfluxDB2
|
22
|
+
module Plugin
|
23
|
+
module Fluent
|
24
|
+
VERSION = '1.5.0'.freeze
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,443 @@
|
|
1
|
+
# The MIT License
|
2
|
+
#
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
4
|
+
# of this software and associated documentation files (the "Software"), to deal
|
5
|
+
# in the Software without restriction, including without limitation the rights
|
6
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
7
|
+
# copies of the Software, and to permit persons to whom the Software is
|
8
|
+
# furnished to do so, subject to the following conditions:
|
9
|
+
#
|
10
|
+
# The above copyright notice and this permission notice shall be included in
|
11
|
+
# all copies or substantial portions of the Software.
|
12
|
+
#
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
19
|
+
# THE SOFTWARE.
|
20
|
+
|
21
|
+
require 'test_helper'
|
22
|
+
|
23
|
+
class InfluxDBOutputTest < Minitest::Test
|
24
|
+
include Fluent::Test::Helpers
|
25
|
+
class MockInfluxDBOutput < InfluxDBOutput
|
26
|
+
attr_reader :client
|
27
|
+
|
28
|
+
def write(chunk)
|
29
|
+
super
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def setup
|
34
|
+
Fluent::Test.setup
|
35
|
+
WebMock.disable_net_connect!
|
36
|
+
end
|
37
|
+
|
38
|
+
def default_config
|
39
|
+
%(
|
40
|
+
@type influxdb2
|
41
|
+
token my-token
|
42
|
+
bucket my-bucket
|
43
|
+
org my-org
|
44
|
+
time_precision ns
|
45
|
+
)
|
46
|
+
end
|
47
|
+
|
48
|
+
def create_driver(conf = default_config)
|
49
|
+
Fluent::Test::Driver::Output.new(MockInfluxDBOutput).configure(conf)
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_configuration
|
53
|
+
driver = create_driver
|
54
|
+
driver.run(default_tag: 'input.influxdb2') do
|
55
|
+
end
|
56
|
+
assert_equal driver.instance.multi_workers_ready?, true
|
57
|
+
end
|
58
|
+
|
59
|
+
def test_time_precision_parameter
|
60
|
+
refute_nil create_driver
|
61
|
+
conf = %(
|
62
|
+
@type influxdb2
|
63
|
+
token my-token
|
64
|
+
bucket my-bucket
|
65
|
+
org my-org
|
66
|
+
time_precision %s
|
67
|
+
)
|
68
|
+
refute_nil create_driver(format(conf, 'ns'))
|
69
|
+
refute_nil create_driver(format(conf, 'us'))
|
70
|
+
refute_nil create_driver(format(conf, 'ms'))
|
71
|
+
refute_nil create_driver(format(conf, 's'))
|
72
|
+
error = assert_raises Fluent::ConfigError do
|
73
|
+
create_driver(format(conf, 'h'))
|
74
|
+
end
|
75
|
+
|
76
|
+
assert_equal 'The time precision h is not supported. You should use: second (s), millisecond (ms), ' \
|
77
|
+
'microsecond (us), or nanosecond (ns).', error.message
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_url_parameter
|
81
|
+
error = assert_raises Fluent::ConfigError do
|
82
|
+
create_driver(%(
|
83
|
+
@type influxdb2
|
84
|
+
url
|
85
|
+
token my-token
|
86
|
+
bucket my-bucket
|
87
|
+
org my-org
|
88
|
+
))
|
89
|
+
end
|
90
|
+
|
91
|
+
assert_equal 'The InfluxDB URL should be defined.', error.message
|
92
|
+
end
|
93
|
+
|
94
|
+
def test_token_parameter
|
95
|
+
error = assert_raises Fluent::ConfigError do
|
96
|
+
create_driver(%(
|
97
|
+
@type influxdb2
|
98
|
+
bucket my-bucket
|
99
|
+
org my-org
|
100
|
+
))
|
101
|
+
end
|
102
|
+
|
103
|
+
assert_equal '\'token\' parameter is required', error.message
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_use_ssl_parameter
|
107
|
+
error = assert_raises Fluent::ConfigError do
|
108
|
+
create_driver(%(
|
109
|
+
@type influxdb2
|
110
|
+
bucket my-bucket
|
111
|
+
org my-org
|
112
|
+
token my-token
|
113
|
+
use_ssl not_bool
|
114
|
+
))
|
115
|
+
end
|
116
|
+
|
117
|
+
assert_equal '\'use_ssl\' parameter is required but nil is specified', error.message
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_bucket_parameter
|
121
|
+
error = assert_raises Fluent::ConfigError do
|
122
|
+
create_driver(%(
|
123
|
+
@type influxdb2
|
124
|
+
token my-token
|
125
|
+
org my-org
|
126
|
+
time_precision ns
|
127
|
+
))
|
128
|
+
end
|
129
|
+
|
130
|
+
assert_equal '\'bucket\' parameter is required', error.message
|
131
|
+
end
|
132
|
+
|
133
|
+
def test_org_parameter
|
134
|
+
error = assert_raises Fluent::ConfigError do
|
135
|
+
create_driver(%(
|
136
|
+
@type influxdb2
|
137
|
+
token my-token
|
138
|
+
bucket my-bucket
|
139
|
+
time_precision ns
|
140
|
+
))
|
141
|
+
end
|
142
|
+
|
143
|
+
assert_equal '\'org\' parameter is required', error.message
|
144
|
+
end
|
145
|
+
|
146
|
+
def test_has_client
|
147
|
+
driver = create_driver
|
148
|
+
driver.run(default_tag: 'input.influxdb2') do
|
149
|
+
end
|
150
|
+
refute_nil driver.instance.client
|
151
|
+
end
|
152
|
+
|
153
|
+
def test_measurement_as_parameter
|
154
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
155
|
+
.to_return(status: 204)
|
156
|
+
driver = create_driver(%(
|
157
|
+
@type influxdb2
|
158
|
+
token my-token
|
159
|
+
bucket my-bucket
|
160
|
+
org my-org
|
161
|
+
measurement h2o
|
162
|
+
time_precision ns
|
163
|
+
))
|
164
|
+
driver.run(default_tag: 'test') do
|
165
|
+
emit_documents(driver)
|
166
|
+
end
|
167
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
168
|
+
times: 1, body: 'h2o level=2i,location="europe" 1444897215000000000')
|
169
|
+
end
|
170
|
+
|
171
|
+
def test_measurement_as_tag
|
172
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
173
|
+
.to_return(status: 204)
|
174
|
+
driver = create_driver(%(
|
175
|
+
@type influxdb2
|
176
|
+
token my-token
|
177
|
+
bucket my-bucket
|
178
|
+
org my-org
|
179
|
+
time_precision ns
|
180
|
+
))
|
181
|
+
driver.run(default_tag: 'h2o_tag') do
|
182
|
+
emit_documents(driver)
|
183
|
+
end
|
184
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
185
|
+
times: 1, body: 'h2o_tag level=2i,location="europe" 1444897215000000000')
|
186
|
+
end
|
187
|
+
|
188
|
+
def test_tag_keys
|
189
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
190
|
+
.to_return(status: 204)
|
191
|
+
driver = create_driver(%(
|
192
|
+
@type influxdb2
|
193
|
+
token my-token
|
194
|
+
bucket my-bucket
|
195
|
+
org my-org
|
196
|
+
tag_keys ["location"]
|
197
|
+
))
|
198
|
+
driver.run(default_tag: 'h2o_tag') do
|
199
|
+
emit_documents(driver)
|
200
|
+
end
|
201
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
202
|
+
times: 1, body: 'h2o_tag,location=europe level=2i 1444897215000000000')
|
203
|
+
end
|
204
|
+
|
205
|
+
def test_tag_fluentd
|
206
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
207
|
+
.to_return(status: 204)
|
208
|
+
driver = create_driver(%(
|
209
|
+
@type influxdb2
|
210
|
+
token my-token
|
211
|
+
bucket my-bucket
|
212
|
+
org my-org
|
213
|
+
measurement h2o
|
214
|
+
tag_keys ["location"]
|
215
|
+
tag_fluentd true
|
216
|
+
))
|
217
|
+
driver.run(default_tag: 'system.logs') do
|
218
|
+
emit_documents(driver)
|
219
|
+
end
|
220
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
221
|
+
times: 1, body: 'h2o,fluentd=system.logs,location=europe level=2i 1444897215000000000')
|
222
|
+
end
|
223
|
+
|
224
|
+
def test_field_keys
|
225
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
226
|
+
.to_return(status: 204)
|
227
|
+
driver = create_driver(%(
|
228
|
+
@type influxdb2
|
229
|
+
token my-token
|
230
|
+
bucket my-bucket
|
231
|
+
org my-org
|
232
|
+
tag_keys ["location"]
|
233
|
+
field_keys ["level"]
|
234
|
+
))
|
235
|
+
driver.run(default_tag: 'h2o_tag') do
|
236
|
+
emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10')
|
237
|
+
end
|
238
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
239
|
+
times: 1, body: 'h2o_tag,location=europe level=2i 1444897215000000000')
|
240
|
+
end
|
241
|
+
|
242
|
+
def test_time_precision_ns
|
243
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
244
|
+
.to_return(status: 204)
|
245
|
+
driver = create_driver(%(
|
246
|
+
@type influxdb2
|
247
|
+
token my-token
|
248
|
+
bucket my-bucket
|
249
|
+
org my-org
|
250
|
+
tag_keys ["version"]
|
251
|
+
time_precision ns
|
252
|
+
))
|
253
|
+
driver.run(default_tag: 'h2o_tag') do
|
254
|
+
emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10')
|
255
|
+
end
|
256
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
257
|
+
times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1444897215000000000')
|
258
|
+
end
|
259
|
+
|
260
|
+
def test_time_integer
|
261
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
262
|
+
.to_return(status: 204)
|
263
|
+
driver = create_driver(%(
|
264
|
+
@type influxdb2
|
265
|
+
token my-token
|
266
|
+
bucket my-bucket
|
267
|
+
org my-org
|
268
|
+
))
|
269
|
+
driver.run(default_tag: 'h2o_tag') do
|
270
|
+
driver.feed(123_465_789, 'location' => 'europe', 'level' => 2, 'version' => 'v.10')
|
271
|
+
end
|
272
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
273
|
+
times: 1, body: 'h2o_tag level=2i,location="europe",version="v.10" 123465789')
|
274
|
+
end
|
275
|
+
|
276
|
+
def test_time_precision_us
|
277
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=us')
|
278
|
+
.to_return(status: 204)
|
279
|
+
driver = create_driver(%(
|
280
|
+
@type influxdb2
|
281
|
+
token my-token
|
282
|
+
bucket my-bucket
|
283
|
+
org my-org
|
284
|
+
tag_keys ["version"]
|
285
|
+
time_precision us
|
286
|
+
))
|
287
|
+
driver.run(default_tag: 'h2o_tag') do
|
288
|
+
emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10')
|
289
|
+
end
|
290
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=us',
|
291
|
+
times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1444897215000000')
|
292
|
+
end
|
293
|
+
|
294
|
+
def test_time_precision_ms
|
295
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ms')
|
296
|
+
.to_return(status: 204)
|
297
|
+
driver = create_driver(%(
|
298
|
+
@type influxdb2
|
299
|
+
token my-token
|
300
|
+
bucket my-bucket
|
301
|
+
org my-org
|
302
|
+
tag_keys ["version"]
|
303
|
+
time_precision ms
|
304
|
+
))
|
305
|
+
driver.run(default_tag: 'h2o_tag') do
|
306
|
+
emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10')
|
307
|
+
end
|
308
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ms',
|
309
|
+
times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1444897215000')
|
310
|
+
end
|
311
|
+
|
312
|
+
def test_time_precision_s
|
313
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=s')
|
314
|
+
.to_return(status: 204)
|
315
|
+
driver = create_driver(%(
|
316
|
+
@type influxdb2
|
317
|
+
token my-token
|
318
|
+
bucket my-bucket
|
319
|
+
org my-org
|
320
|
+
tag_keys ["version"]
|
321
|
+
time_precision s
|
322
|
+
))
|
323
|
+
driver.run(default_tag: 'h2o_tag') do
|
324
|
+
emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10')
|
325
|
+
end
|
326
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=s',
|
327
|
+
times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1444897215')
|
328
|
+
end
|
329
|
+
|
330
|
+
def test_time_key
|
331
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
332
|
+
.to_return(status: 204)
|
333
|
+
driver = create_driver(%(
|
334
|
+
@type influxdb2
|
335
|
+
token my-token
|
336
|
+
bucket my-bucket
|
337
|
+
org my-org
|
338
|
+
tag_keys ["version"]
|
339
|
+
time_key time
|
340
|
+
))
|
341
|
+
driver.run(default_tag: 'h2o_tag') do
|
342
|
+
emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10',
|
343
|
+
'time' => 1_544_897_215_000_000_000)
|
344
|
+
end
|
345
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
346
|
+
times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1544897215000000000')
|
347
|
+
end
|
348
|
+
|
349
|
+
def test_field_cast_to_float
|
350
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
351
|
+
.to_return(status: 204)
|
352
|
+
driver = create_driver(%(
|
353
|
+
@type influxdb2
|
354
|
+
token my-token
|
355
|
+
bucket my-bucket
|
356
|
+
org my-org
|
357
|
+
field_cast_to_float true
|
358
|
+
))
|
359
|
+
driver.run(default_tag: 'h2o_tag') do
|
360
|
+
emit_documents(driver)
|
361
|
+
end
|
362
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
363
|
+
times: 1, body: 'h2o_tag level=2.0,location="europe" 1444897215000000000')
|
364
|
+
end
|
365
|
+
|
366
|
+
def test_nested_field
|
367
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
368
|
+
.to_return(status: 204)
|
369
|
+
driver = create_driver(%(
|
370
|
+
@type influxdb2
|
371
|
+
token my-token
|
372
|
+
bucket my-bucket
|
373
|
+
org my-org
|
374
|
+
time_precision ns
|
375
|
+
))
|
376
|
+
driver.run(default_tag: 'h2o_tag') do
|
377
|
+
emit_documents(driver, 'location' => 'europe', 'level' => 2,
|
378
|
+
'nest' => { 'key' => 'nested value', 'deep' => { 'deep-key' => 'deep-value' }, 'a' => 25 })
|
379
|
+
end
|
380
|
+
body = 'h2o_tag level=2i,location="europe",nest.a=25i,nest.deep.deep-key="deep-value",nest.key="nested value" ' \
|
381
|
+
'1444897215000000000'
|
382
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
383
|
+
times: 1, body: body)
|
384
|
+
end
|
385
|
+
|
386
|
+
def test_kubernetes_structure
|
387
|
+
record = {
|
388
|
+
'docker' => { 'container_id' => '7ee0723e90d13df5ade6f5d524f23474461fcfeb48a90630d8b02b13c741550b' },
|
389
|
+
'kubernetes' => { 'container_name' => 'fluentd',
|
390
|
+
'namespace_name' => 'default',
|
391
|
+
'pod_name' => 'fluentd-49xk2',
|
392
|
+
'container_image' => 'rawkode/fluentd:1',
|
393
|
+
'container_image_id' =>
|
394
|
+
'docker://sha256:90c288b8a09cc6ae98b04078afb10d9c380c0603a47745403461435073460f97',
|
395
|
+
'pod_id' => 'c15ab1cb-0773-4ad7-a58b-f791ab34c62f',
|
396
|
+
'host' => 'minikube',
|
397
|
+
'labels' => {
|
398
|
+
'controller-revision-hash' => '57748799f7',
|
399
|
+
'pod-template-generation' => 2,
|
400
|
+
'app_kubernetes_io/instance' => 'fluentd',
|
401
|
+
'app_kubernetes_io/name' => 'fluentd'
|
402
|
+
},
|
403
|
+
'master_url' => 'https://10.96.0.1:443/api',
|
404
|
+
'namespace_id' => '2b0bc75c-323a-4f04-9eec-02255a8d0044' },
|
405
|
+
'log' => '2020-06-17 13:19:42 +0000 [info]: #0 [filter_kube_metadata] stats - namespace_cache_size: 2, '\
|
406
|
+
'pod_cache_size: 6, pod_cache_watch_updates: 1, namespace_cache_api_updates: 6, '\
|
407
|
+
'pod_cache_api_updates: 6, id_cache_miss: 6\n',
|
408
|
+
'stream' => 'stdout'
|
409
|
+
}
|
410
|
+
|
411
|
+
stub_request(:any, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
|
412
|
+
.to_return(status: 204)
|
413
|
+
driver = create_driver(%(
|
414
|
+
@type influxdb2
|
415
|
+
token my-token
|
416
|
+
bucket my-bucket
|
417
|
+
org my-org
|
418
|
+
time_precision ns
|
419
|
+
))
|
420
|
+
driver.run(default_tag: 'h2o_tag') do
|
421
|
+
emit_documents(driver, record)
|
422
|
+
end
|
423
|
+
body = 'h2o_tag docker.container_id="7ee0723e90d13df5ade6f5d524f23474461fcfeb48a90630d8b02b13c741550b",'\
|
424
|
+
'kubernetes.container_image="rawkode/fluentd:1",'\
|
425
|
+
'kubernetes.container_image_id="docker://sha256:90c288b8a09cc6ae98b04078afb10d9c380c0603a47745403461435073460f97",'\
|
426
|
+
'kubernetes.container_name="fluentd",kubernetes.host="minikube",'\
|
427
|
+
'kubernetes.labels.app_kubernetes_io/instance="fluentd",kubernetes.labels.app_kubernetes_io/name="fluentd",'\
|
428
|
+
'kubernetes.labels.controller-revision-hash="57748799f7",kubernetes.labels.pod-template-generation=2i,'\
|
429
|
+
'kubernetes.master_url="https://10.96.0.1:443/api",kubernetes.namespace_id="2b0bc75c-323a-4f04-9eec-02255a8d0044",'\
|
430
|
+
'kubernetes.namespace_name="default",kubernetes.pod_id="c15ab1cb-0773-4ad7-a58b-f791ab34c62f",'\
|
431
|
+
'kubernetes.pod_name="fluentd-49xk2",log="2020-06-17 13:19:42 +0000 [info]: #0 [filter_kube_metadata] stats - '\
|
432
|
+
'namespace_cache_size: 2, pod_cache_size: 6, pod_cache_watch_updates: 1, namespace_cache_api_updates: 6, '\
|
433
|
+
'pod_cache_api_updates: 6, id_cache_miss: 6\\\n",stream="stdout" 1444897215000000000'
|
434
|
+
assert_requested(:post, 'https://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
|
435
|
+
times: 1, body: body)
|
436
|
+
end
|
437
|
+
|
438
|
+
def emit_documents(driver, data = { 'location' => 'europe', 'level' => 2 })
|
439
|
+
time = event_time('2015-10-15 8:20:15 UTC')
|
440
|
+
driver.feed(time, data)
|
441
|
+
time
|
442
|
+
end
|
443
|
+
end
|