fluent-plugin-input-opensearch 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.coveralls.yml +1 -0
- data/.editorconfig +9 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +29 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
- data/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +9 -0
- data/.github/workflows/coverage.yaml +22 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/.gitignore +18 -0
- data/CONTRIBUTING.md +24 -0
- data/Gemfile +10 -0
- data/History.md +67 -0
- data/LICENSE.txt +201 -0
- data/README.OpenSearchGenID.md +116 -0
- data/README.OpenSearchInput.md +314 -0
- data/README.Troubleshooting.md +482 -0
- data/README.md +1622 -0
- data/Rakefile +37 -0
- data/fluent-plugin-opensearch.gemspec +39 -0
- data/gemfiles/Gemfile.elasticsearch.v6 +12 -0
- data/lib/fluent/log-ext.rb +64 -0
- data/lib/fluent/plugin/filter_opensearch_genid.rb +103 -0
- data/lib/fluent/plugin/in_opensearch.rb +410 -0
- data/lib/fluent/plugin/oj_serializer.rb +48 -0
- data/lib/fluent/plugin/opensearch_constants.rb +39 -0
- data/lib/fluent/plugin/opensearch_error.rb +31 -0
- data/lib/fluent/plugin/opensearch_error_handler.rb +182 -0
- data/lib/fluent/plugin/opensearch_fallback_selector.rb +36 -0
- data/lib/fluent/plugin/opensearch_index_template.rb +155 -0
- data/lib/fluent/plugin/opensearch_simple_sniffer.rb +36 -0
- data/lib/fluent/plugin/opensearch_tls.rb +96 -0
- data/lib/fluent/plugin/out_opensearch.rb +1158 -0
- data/lib/fluent/plugin/out_opensearch_data_stream.rb +229 -0
- data/test/helper.rb +60 -0
- data/test/plugin/datastream_template.json +4 -0
- data/test/plugin/test_alias_template.json +9 -0
- data/test/plugin/test_filter_opensearch_genid.rb +241 -0
- data/test/plugin/test_in_opensearch.rb +500 -0
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_oj_serializer.rb +45 -0
- data/test/plugin/test_opensearch_error_handler.rb +770 -0
- data/test/plugin/test_opensearch_fallback_selector.rb +100 -0
- data/test/plugin/test_opensearch_tls.rb +171 -0
- data/test/plugin/test_out_opensearch.rb +3980 -0
- data/test/plugin/test_out_opensearch_data_stream.rb +746 -0
- data/test/plugin/test_template.json +23 -0
- data/test/test_log-ext.rb +61 -0
- metadata +291 -0
@@ -0,0 +1,229 @@
|
|
1
|
+
|
2
|
+
require_relative 'out_opensearch'
|
3
|
+
|
4
|
+
module Fluent::Plugin
|
5
|
+
class OpenSearchOutputDataStream < OpenSearchOutput
|
6
|
+
|
7
|
+
Fluent::Plugin.register_output('opensearch_data_stream', self)
|
8
|
+
|
9
|
+
helpers :event_emitter
|
10
|
+
|
11
|
+
config_param :data_stream_name, :string
|
12
|
+
config_param :data_stream_template_name, :string, :default => nil
|
13
|
+
# OpenSearch 1.0 or later always support new style of index template.
|
14
|
+
config_set_default :use_legacy_template, false
|
15
|
+
|
16
|
+
INVALID_START_CHRACTERS = ["-", "_", "+", "."]
|
17
|
+
INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
|
18
|
+
|
19
|
+
def configure(conf)
|
20
|
+
super
|
21
|
+
|
22
|
+
@data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
|
23
|
+
|
24
|
+
# ref. https://opensearch.org/docs/latest/opensearch/data-streams/
|
25
|
+
unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
|
26
|
+
validate_data_stream_parameters
|
27
|
+
else
|
28
|
+
@use_placeholder = true
|
29
|
+
@data_stream_names = []
|
30
|
+
end
|
31
|
+
|
32
|
+
unless @use_placeholder
|
33
|
+
begin
|
34
|
+
@data_stream_names = [@data_stream_name]
|
35
|
+
retry_operate(@max_retry_putting_template,
|
36
|
+
@fail_on_putting_template_retry_exceed,
|
37
|
+
@catch_transport_exception_on_retry) do
|
38
|
+
create_index_template(@data_stream_name, @data_stream_template_name)
|
39
|
+
end
|
40
|
+
rescue => e
|
41
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def validate_data_stream_parameters
|
47
|
+
{"data_stream_name" => @data_stream_name,
|
48
|
+
"data_stream_template_name" => @data_stream_template_name}.each do |parameter, value|
|
49
|
+
unless valid_data_stream_parameters?(value)
|
50
|
+
unless start_with_valid_characters?(value)
|
51
|
+
if not_dots?(value)
|
52
|
+
raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
|
53
|
+
else
|
54
|
+
raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
unless valid_characters?(value)
|
58
|
+
raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
|
59
|
+
end
|
60
|
+
unless lowercase_only?(value)
|
61
|
+
raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
|
62
|
+
end
|
63
|
+
if value.bytes.size > 255
|
64
|
+
raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def create_index_template(datastream_name, template_name, host = nil)
|
71
|
+
# Create index template from file
|
72
|
+
if !dry_run?
|
73
|
+
if @template_file
|
74
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
75
|
+
template_installation_actual(template_name, @customize_template, @application_name, datastream_name, host)
|
76
|
+
else # Create default index template
|
77
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
78
|
+
body = {
|
79
|
+
"index_patterns" => ["#{datastream_name}*"],
|
80
|
+
"data_stream" => {},
|
81
|
+
}
|
82
|
+
|
83
|
+
params = {
|
84
|
+
name: template_name,
|
85
|
+
body: body
|
86
|
+
}
|
87
|
+
retry_operate(@max_retry_putting_template,
|
88
|
+
@fail_on_putting_template_retry_exceed,
|
89
|
+
@catch_transport_exception_on_retry) do
|
90
|
+
client(host).indices.put_index_template(params)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def data_stream_exist?(datastream_name, host = nil)
|
97
|
+
params = {
|
98
|
+
name: datastream_name
|
99
|
+
}
|
100
|
+
begin
|
101
|
+
# TODO: Use X-Pack equivalent performing DataStream operation method on the following line
|
102
|
+
response = client(host).perform_request('GET', "/_data_stream/#{datastream_name}", {}, params)
|
103
|
+
return (not response.is_a?(OpenSearch::Transport::Transport::Errors::NotFound))
|
104
|
+
rescue OpenSearch::Transport::Transport::Errors::NotFound => e
|
105
|
+
log.info "Specified data stream does not exist. Will be created: <#{e}>"
|
106
|
+
return false
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def template_exists?(name, host = nil)
|
111
|
+
if @use_legacy_template
|
112
|
+
client(host).indices.get_template(:name => name)
|
113
|
+
else
|
114
|
+
client(host).indices.get_index_template(:name => name)
|
115
|
+
end
|
116
|
+
return true
|
117
|
+
rescue OpenSearch::Transport::Transport::Errors::NotFound
|
118
|
+
return false
|
119
|
+
end
|
120
|
+
|
121
|
+
def valid_data_stream_parameters?(data_stream_parameter)
|
122
|
+
lowercase_only?(data_stream_parameter) and
|
123
|
+
valid_characters?(data_stream_parameter) and
|
124
|
+
start_with_valid_characters?(data_stream_parameter) and
|
125
|
+
not_dots?(data_stream_parameter) and
|
126
|
+
data_stream_parameter.bytes.size <= 255
|
127
|
+
end
|
128
|
+
|
129
|
+
def lowercase_only?(data_stream_parameter)
|
130
|
+
data_stream_parameter.downcase == data_stream_parameter
|
131
|
+
end
|
132
|
+
|
133
|
+
def valid_characters?(data_stream_parameter)
|
134
|
+
not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
|
135
|
+
end
|
136
|
+
|
137
|
+
def start_with_valid_characters?(data_stream_parameter)
|
138
|
+
not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
|
139
|
+
end
|
140
|
+
|
141
|
+
def not_dots?(data_stream_parameter)
|
142
|
+
not (data_stream_parameter == "." or data_stream_parameter == "..")
|
143
|
+
end
|
144
|
+
|
145
|
+
def client_library_version
|
146
|
+
OpenSearch::VERSION
|
147
|
+
end
|
148
|
+
|
149
|
+
def multi_workers_ready?
|
150
|
+
true
|
151
|
+
end
|
152
|
+
|
153
|
+
def write(chunk)
|
154
|
+
data_stream_name = @data_stream_name
|
155
|
+
data_stream_template_name = @data_stream_template_name
|
156
|
+
host = nil
|
157
|
+
if @use_placeholder
|
158
|
+
host = if @hosts
|
159
|
+
extract_placeholders(@hosts, chunk)
|
160
|
+
else
|
161
|
+
extract_placeholders(@host, chunk)
|
162
|
+
end
|
163
|
+
data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
|
164
|
+
data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
|
165
|
+
begin
|
166
|
+
create_index_template(data_stream_name, data_stream_template_name, host)
|
167
|
+
rescue => e
|
168
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
bulk_message = ""
|
173
|
+
headers = {
|
174
|
+
CREATE_OP => {}
|
175
|
+
}
|
176
|
+
tag = chunk.metadata.tag
|
177
|
+
chunk.msgpack_each do |time, record|
|
178
|
+
next unless record.is_a? Hash
|
179
|
+
begin
|
180
|
+
if record.has_key?(TIMESTAMP_FIELD)
|
181
|
+
rts = record[TIMESTAMP_FIELD]
|
182
|
+
dt = parse_time(rts, time, tag)
|
183
|
+
elsif record.has_key?(@time_key)
|
184
|
+
rts = record[@time_key]
|
185
|
+
dt = parse_time(rts, time, tag)
|
186
|
+
else
|
187
|
+
dt = Time.at(time).to_datetime
|
188
|
+
end
|
189
|
+
record.merge!({"@timestamp" => dt.iso8601(@time_precision)})
|
190
|
+
if @include_tag_key
|
191
|
+
record[@tag_key] = tag
|
192
|
+
end
|
193
|
+
if @remove_keys
|
194
|
+
@remove_keys.each { |key| record.delete(key) }
|
195
|
+
end
|
196
|
+
bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
|
197
|
+
rescue => e
|
198
|
+
emit_error_label_event do
|
199
|
+
router.emit_error_event(tag, time, record, e)
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
params = {
|
205
|
+
index: data_stream_name,
|
206
|
+
body: bulk_message
|
207
|
+
}
|
208
|
+
begin
|
209
|
+
response = client(host).bulk(params)
|
210
|
+
if response['errors']
|
211
|
+
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
|
212
|
+
end
|
213
|
+
rescue => e
|
214
|
+
raise RecoverableRequestFailure, "could not push logs to OpenSearch cluster (#{data_stream_name}): #{e.message}"
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
def append_record_to_messages(op, meta, header, record, msgs)
|
219
|
+
header[CREATE_OP] = meta
|
220
|
+
msgs << @dump_proc.call(header) << BODY_DELIMITER
|
221
|
+
msgs << @dump_proc.call(record) << BODY_DELIMITER
|
222
|
+
msgs
|
223
|
+
end
|
224
|
+
|
225
|
+
def retry_stream_retryable?
|
226
|
+
@buffer.storable?
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
#
|
4
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
5
|
+
# this file be licensed under the Apache-2.0 license or a
|
6
|
+
# compatible open source license.
|
7
|
+
#
|
8
|
+
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
9
|
+
# GitHub history for details.
|
10
|
+
#
|
11
|
+
# Licensed to Uken Inc. under one or more contributor
|
12
|
+
# license agreements. See the NOTICE file distributed with
|
13
|
+
# this work for additional information regarding copyright
|
14
|
+
# ownership. Uken Inc. licenses this file to you under
|
15
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
16
|
+
# not use this file except in compliance with the License.
|
17
|
+
# You may obtain a copy of the License at
|
18
|
+
#
|
19
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
20
|
+
#
|
21
|
+
# Unless required by applicable law or agreed to in writing,
|
22
|
+
# software distributed under the License is distributed on an
|
23
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
24
|
+
# KIND, either express or implied. See the License for the
|
25
|
+
# specific language governing permissions and limitations
|
26
|
+
# under the License.
|
27
|
+
|
28
|
+
require 'simplecov'
|
29
|
+
require 'simplecov'
|
30
|
+
require 'simplecov-lcov'
|
31
|
+
|
32
|
+
SimpleCov::Formatter::LcovFormatter.config do |config|
|
33
|
+
config.report_with_single_file = true
|
34
|
+
config.single_report_path = 'coverage/lcov.info'
|
35
|
+
end
|
36
|
+
|
37
|
+
SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
|
38
|
+
SimpleCov::Formatter::HTMLFormatter,
|
39
|
+
SimpleCov::Formatter::LcovFormatter
|
40
|
+
])
|
41
|
+
|
42
|
+
SimpleCov.start do
|
43
|
+
add_filter do |src|
|
44
|
+
!(src.filename =~ /^#{SimpleCov.root}\/lib/)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# needs to be after simplecov but before test/unit, because fluentd sets default
|
49
|
+
# encoding to ASCII-8BIT, but coverall might load git data which could contain a
|
50
|
+
# UTF-8 character
|
51
|
+
at_exit do
|
52
|
+
Encoding.default_internal = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_internal)
|
53
|
+
Encoding.default_external = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_external)
|
54
|
+
end
|
55
|
+
|
56
|
+
require 'test/unit'
|
57
|
+
require 'fluent/test'
|
58
|
+
|
59
|
+
require 'webmock/test_unit'
|
60
|
+
WebMock.disable_net_connect!
|
@@ -0,0 +1,241 @@
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
2
|
+
#
|
3
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
+
# this file be licensed under the Apache-2.0 license or a
|
5
|
+
# compatible open source license.
|
6
|
+
#
|
7
|
+
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
+
# GitHub history for details.
|
9
|
+
#
|
10
|
+
# Licensed to Uken Inc. under one or more contributor
|
11
|
+
# license agreements. See the NOTICE file distributed with
|
12
|
+
# this work for additional information regarding copyright
|
13
|
+
# ownership. Uken Inc. licenses this file to you under
|
14
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
+
# not use this file except in compliance with the License.
|
16
|
+
# You may obtain a copy of the License at
|
17
|
+
#
|
18
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
+
#
|
20
|
+
# Unless required by applicable law or agreed to in writing,
|
21
|
+
# software distributed under the License is distributed on an
|
22
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
+
# KIND, either express or implied. See the License for the
|
24
|
+
# specific language governing permissions and limitations
|
25
|
+
# under the License.
|
26
|
+
|
27
|
+
require_relative '../helper'
|
28
|
+
require 'date'
|
29
|
+
require 'fluent/test/helpers'
|
30
|
+
require 'json'
|
31
|
+
require 'fluent/test/driver/filter'
|
32
|
+
require 'flexmock/test_unit'
|
33
|
+
require 'fluent/plugin/filter_opensearch_genid'
|
34
|
+
|
35
|
+
class OpenSearchGenidFilterTest < Test::Unit::TestCase
|
36
|
+
include FlexMock::TestCase
|
37
|
+
include Fluent::Test::Helpers
|
38
|
+
|
39
|
+
def setup
|
40
|
+
Fluent::Test.setup
|
41
|
+
end
|
42
|
+
|
43
|
+
def create_driver(conf='')
|
44
|
+
Fluent::Test::Driver::Filter.new(Fluent::Plugin::OpenSearchGenidFilter).configure(conf)
|
45
|
+
end
|
46
|
+
|
47
|
+
test "invalid configuration" do
|
48
|
+
assert_raise(Fluent::ConfigError) do
|
49
|
+
create_driver("use_record_as_seed true")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def sample_record
|
54
|
+
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_configure
|
58
|
+
d = create_driver
|
59
|
+
assert_equal '_hash', d.instance.hash_id_key
|
60
|
+
end
|
61
|
+
|
62
|
+
data("default" => {"hash_id_key" => "_hash"},
|
63
|
+
"custom_key" => {"hash_id_key" => "_edited"},
|
64
|
+
)
|
65
|
+
def test_filter(data)
|
66
|
+
d = create_driver("hash_id_key #{data["hash_id_key"]}")
|
67
|
+
flexmock(SecureRandom).should_receive(:uuid)
|
68
|
+
.and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
|
69
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
70
|
+
d.run(default_tag: 'test') do
|
71
|
+
d.feed(time, sample_record)
|
72
|
+
end
|
73
|
+
assert_equal(Base64.strict_encode64(SecureRandom.uuid),
|
74
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
75
|
+
end
|
76
|
+
|
77
|
+
class UseRecordAsSeedTest < self
|
78
|
+
data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
|
79
|
+
"sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
|
80
|
+
"sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
|
81
|
+
"sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
|
82
|
+
)
|
83
|
+
def test_simple(data)
|
84
|
+
hash_type, expected = data
|
85
|
+
d = create_driver(%[
|
86
|
+
use_record_as_seed true
|
87
|
+
record_keys age,parent_id,routing_id,custom_key
|
88
|
+
hash_type #{hash_type}
|
89
|
+
])
|
90
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
91
|
+
d.run(default_tag: 'test') do
|
92
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
93
|
+
end
|
94
|
+
assert_equal(expected,
|
95
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
96
|
+
end
|
97
|
+
|
98
|
+
data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
|
99
|
+
"sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
|
100
|
+
"sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
|
101
|
+
"sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
|
102
|
+
def test_record_with_tag(data)
|
103
|
+
hash_type, expected = data
|
104
|
+
d = create_driver(%[
|
105
|
+
use_record_as_seed true
|
106
|
+
record_keys age,parent_id,routing_id,custom_key
|
107
|
+
hash_type #{hash_type}
|
108
|
+
include_tag_in_seed true
|
109
|
+
])
|
110
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
111
|
+
d.run(default_tag: 'test.fluentd') do
|
112
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
113
|
+
end
|
114
|
+
assert_equal(expected,
|
115
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
116
|
+
end
|
117
|
+
|
118
|
+
data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
|
119
|
+
"sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
|
120
|
+
"sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
|
121
|
+
"sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
|
122
|
+
def test_record_with_time(data)
|
123
|
+
hash_type, expected = data
|
124
|
+
d = create_driver(%[
|
125
|
+
use_record_as_seed true
|
126
|
+
record_keys age,parent_id,routing_id,custom_key
|
127
|
+
hash_type #{hash_type}
|
128
|
+
include_time_in_seed true
|
129
|
+
])
|
130
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
131
|
+
d.run(default_tag: 'test.fluentd') do
|
132
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
133
|
+
end
|
134
|
+
assert_equal(expected,
|
135
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
136
|
+
end
|
137
|
+
|
138
|
+
data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
|
139
|
+
"sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
|
140
|
+
"sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
|
141
|
+
"sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
|
142
|
+
def test_record_with_tag_and_time
|
143
|
+
hash_type, expected = data
|
144
|
+
d = create_driver(%[
|
145
|
+
use_record_as_seed true
|
146
|
+
record_keys age,parent_id,routing_id,custom_key
|
147
|
+
hash_type #{hash_type}
|
148
|
+
include_tag_in_seed true
|
149
|
+
include_time_in_seed true
|
150
|
+
])
|
151
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
152
|
+
d.run(default_tag: 'test.fluentd') do
|
153
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
154
|
+
end
|
155
|
+
assert_equal(expected,
|
156
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
class UseEntireRecordAsSeedTest < self
|
161
|
+
data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
|
162
|
+
"sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
|
163
|
+
"sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
|
164
|
+
"sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
|
165
|
+
def test_record
|
166
|
+
hash_type, expected = data
|
167
|
+
d = create_driver(%[
|
168
|
+
use_record_as_seed true
|
169
|
+
use_entire_record true
|
170
|
+
hash_type #{hash_type}
|
171
|
+
])
|
172
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
173
|
+
d.run(default_tag: 'test.fluentd') do
|
174
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
175
|
+
end
|
176
|
+
assert_equal(expected,
|
177
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
178
|
+
end
|
179
|
+
|
180
|
+
data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
|
181
|
+
"sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
|
182
|
+
"sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
|
183
|
+
"sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
|
184
|
+
def test_record_with_tag
|
185
|
+
hash_type, expected = data
|
186
|
+
d = create_driver(%[
|
187
|
+
use_record_as_seed true
|
188
|
+
use_entire_record true
|
189
|
+
hash_type #{hash_type}
|
190
|
+
include_tag_in_seed true
|
191
|
+
])
|
192
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
193
|
+
d.run(default_tag: 'test.fluentd') do
|
194
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
195
|
+
end
|
196
|
+
assert_equal(expected,
|
197
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
198
|
+
end
|
199
|
+
|
200
|
+
data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
|
201
|
+
"sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
|
202
|
+
"sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
|
203
|
+
"sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
|
204
|
+
def test_record_with_time
|
205
|
+
hash_type, expected = data
|
206
|
+
d = create_driver(%[
|
207
|
+
use_record_as_seed true
|
208
|
+
use_entire_record true
|
209
|
+
hash_type #{hash_type}
|
210
|
+
include_time_in_seed true
|
211
|
+
])
|
212
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
213
|
+
d.run(default_tag: 'test.fluentd') do
|
214
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
215
|
+
end
|
216
|
+
assert_equal(expected,
|
217
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
218
|
+
end
|
219
|
+
|
220
|
+
data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
|
221
|
+
"sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
|
222
|
+
"sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
|
223
|
+
"sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
|
224
|
+
def test_record_with_tag_and_time
|
225
|
+
hash_type, expected = data
|
226
|
+
d = create_driver(%[
|
227
|
+
use_record_as_seed true
|
228
|
+
use_entire_record true
|
229
|
+
hash_type #{hash_type}
|
230
|
+
include_tag_in_seed true
|
231
|
+
include_time_in_seed true
|
232
|
+
])
|
233
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
234
|
+
d.run(default_tag: 'test.fluentd') do
|
235
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
236
|
+
end
|
237
|
+
assert_equal(expected,
|
238
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|