fluent-plugin-elasticsearch-dext 5.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.coveralls.yml +2 -0
- data/.editorconfig +9 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/.gitignore +18 -0
- data/.travis.yml +40 -0
- data/CONTRIBUTING.md +24 -0
- data/Gemfile +11 -0
- data/History.md +553 -0
- data/ISSUE_TEMPLATE.md +30 -0
- data/LICENSE.txt +201 -0
- data/PULL_REQUEST_TEMPLATE.md +10 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.ElasticsearchInput.md +293 -0
- data/README.Troubleshooting.md +601 -0
- data/README.md +1467 -0
- data/Rakefile +11 -0
- data/appveyor.yml +20 -0
- data/fluent-plugin-elasticsearch.gemspec +35 -0
- data/gemfiles/Gemfile.elasticsearch.v6 +12 -0
- data/lib/fluent/log-ext.rb +38 -0
- data/lib/fluent/plugin/default-ilm-policy.json +14 -0
- data/lib/fluent/plugin/elasticsearch_constants.rb +13 -0
- data/lib/fluent/plugin/elasticsearch_error.rb +5 -0
- data/lib/fluent/plugin/elasticsearch_error_handler.rb +129 -0
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +9 -0
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +67 -0
- data/lib/fluent/plugin/elasticsearch_index_template.rb +211 -0
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +10 -0
- data/lib/fluent/plugin/elasticsearch_tls.rb +70 -0
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +77 -0
- data/lib/fluent/plugin/in_elasticsearch.rb +325 -0
- data/lib/fluent/plugin/oj_serializer.rb +22 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +1108 -0
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +218 -0
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +282 -0
- data/test/helper.rb +24 -0
- data/test/plugin/test_alias_template.json +9 -0
- data/test/plugin/test_elasticsearch_error_handler.rb +646 -0
- data/test/plugin/test_elasticsearch_fallback_selector.rb +74 -0
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +66 -0
- data/test/plugin/test_elasticsearch_tls.rb +145 -0
- data/test/plugin/test_filter_elasticsearch_genid.rb +215 -0
- data/test/plugin/test_in_elasticsearch.rb +459 -0
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_oj_serializer.rb +19 -0
- data/test/plugin/test_out_elasticsearch.rb +5688 -0
- data/test/plugin/test_out_elasticsearch_data_stream.rb +337 -0
- data/test/plugin/test_out_elasticsearch_dynamic.rb +1134 -0
- data/test/plugin/test_template.json +23 -0
- data/test/test_log-ext.rb +35 -0
- metadata +236 -0
@@ -0,0 +1,74 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/output'
|
3
|
+
require 'fluent/plugin/out_elasticsearch'
|
4
|
+
|
5
|
+
class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
6
|
+
attr_accessor :index_cmds
|
7
|
+
|
8
|
+
def setup
|
9
|
+
Fluent::Test.setup
|
10
|
+
@driver = nil
|
11
|
+
log = Fluent::Engine.log
|
12
|
+
log.out.logs.slice!(0, log.out.logs.length)
|
13
|
+
end
|
14
|
+
|
15
|
+
def stub_elastic(url="http://localhost:9200/_bulk")
|
16
|
+
stub_request(:post, url).with do |req|
|
17
|
+
@index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
|
22
|
+
body ="{\"version\":{\"number\":\"#{version}\"}}"
|
23
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
24
|
+
end
|
25
|
+
|
26
|
+
def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
|
27
|
+
stub_request(:get, url).to_return(:status => [404, "Not Found"])
|
28
|
+
end
|
29
|
+
|
30
|
+
def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
|
31
|
+
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
|
32
|
+
end
|
33
|
+
|
34
|
+
def sample_record(content={})
|
35
|
+
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
|
36
|
+
end
|
37
|
+
|
38
|
+
def driver(conf='')
|
39
|
+
@driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
|
40
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
41
|
+
if !defined?(Fluent::Plugin::Output)
|
42
|
+
def format(tag, time, record)
|
43
|
+
[time, record].to_msgpack
|
44
|
+
end
|
45
|
+
end
|
46
|
+
}.configure(conf)
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_fallback_on_info
|
50
|
+
stub_elastic_info_not_found("http://localhost:9202/")
|
51
|
+
stub_elastic_info_unavailable("http://localhost:9201/")
|
52
|
+
stub_elastic_info
|
53
|
+
stub_elastic
|
54
|
+
config = %[
|
55
|
+
hosts localhost:9202,localhost:9201,localhost:9200
|
56
|
+
selector_class_name Fluent::Plugin::ElasticseatchFallbackSelector
|
57
|
+
@log_level debug
|
58
|
+
with_transporter_log true
|
59
|
+
reload_connections true
|
60
|
+
reload_after 10
|
61
|
+
catch_transport_exception_on_retry false # For fallback testing
|
62
|
+
]
|
63
|
+
assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
|
64
|
+
driver(config)
|
65
|
+
end
|
66
|
+
driver.run(default_tag: 'test') do
|
67
|
+
driver.feed(sample_record)
|
68
|
+
end
|
69
|
+
assert_equal(2, index_cmds.length)
|
70
|
+
assert_equal("fluentd", index_cmds.first['index']['_index'])
|
71
|
+
end
|
72
|
+
|
73
|
+
# TODO: on feed phase test case
|
74
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'elasticsearch'
|
3
|
+
require 'fluent/plugin/elasticsearch_index_lifecycle_management'
|
4
|
+
|
5
|
+
class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
|
6
|
+
include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
7
|
+
|
8
|
+
def setup
|
9
|
+
begin
|
10
|
+
require "elasticsearch/xpack"
|
11
|
+
rescue LoadError
|
12
|
+
omit "ILM testcase needs elasticsearch-xpack gem."
|
13
|
+
end
|
14
|
+
if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
|
15
|
+
omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
|
16
|
+
end
|
17
|
+
Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
|
18
|
+
def client
|
19
|
+
Elasticsearch::Client.new url: 'localhost:9200'
|
20
|
+
end
|
21
|
+
def log
|
22
|
+
log_device = Fluent::Test::DummyLogDevice.new
|
23
|
+
dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
|
24
|
+
logger = ServerEngine::DaemonLogger.new(log_device, dl_opts)
|
25
|
+
Fluent::Log.new(logger)
|
26
|
+
end
|
27
|
+
CODE
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_xpack_info
|
31
|
+
stub_request(:get, "http://localhost:9200/_xpack").
|
32
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
33
|
+
expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
|
34
|
+
assert_equal(expected, xpack_info)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_verify_ilm_working
|
38
|
+
stub_request(:get, "http://localhost:9200/_xpack").
|
39
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
40
|
+
assert_nothing_raised { verify_ilm_working }
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_ilm_policy_doesnt_exists
|
44
|
+
stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
|
45
|
+
to_return(:status => 404, :body => "", :headers => {})
|
46
|
+
assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_ilm_policy_exists
|
50
|
+
stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
|
51
|
+
to_return(:status => 200, :body => "", :headers => {})
|
52
|
+
assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_create_ilm_policy
|
56
|
+
stub_request(:get, "http://localhost:9200/_ilm/policy/fluent-policy").
|
57
|
+
to_return(:status => 404, :body => "", :headers => {})
|
58
|
+
stub_request(:put, "http://localhost:9200/_ilm/policy/fluent-policy").
|
59
|
+
with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
|
60
|
+
:headers => {'Content-Type'=>'application/json'}).
|
61
|
+
to_return(:status => 200, :body => "", :headers => {})
|
62
|
+
create_ilm_policy("fluent-policy")
|
63
|
+
|
64
|
+
assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,145 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/output'
|
3
|
+
require 'fluent/plugin/output'
|
4
|
+
require 'fluent/plugin/elasticsearch_tls'
|
5
|
+
|
6
|
+
class TestElasticsearchTLS < Test::Unit::TestCase
|
7
|
+
|
8
|
+
class TestTLSModuleOutput < Fluent::Plugin::Output
|
9
|
+
include Fluent::Plugin::ElasticsearchTLS
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
super
|
13
|
+
@emit_streams = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def write(chunk)
|
17
|
+
es = Fluent::ArrayEventStream.new
|
18
|
+
chunk.each do |time, record|
|
19
|
+
es.add(time, record)
|
20
|
+
end
|
21
|
+
@emit_streams << [tag, es]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
setup do
|
26
|
+
Fluent::Test.setup
|
27
|
+
@use_tls_minmax_version = begin
|
28
|
+
map = {
|
29
|
+
TLSv1: OpenSSL::SSL::TLS1_VERSION,
|
30
|
+
TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
|
31
|
+
TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
|
32
|
+
}
|
33
|
+
map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
|
34
|
+
true
|
35
|
+
rescue NameError
|
36
|
+
false
|
37
|
+
end
|
38
|
+
@enabled_tlsv1_3 = begin
|
39
|
+
map = {TLSv1_3: OpenSSL::SSL::TLS1_3_VERSION}
|
40
|
+
true
|
41
|
+
rescue NameError
|
42
|
+
false
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def driver(conf='')
|
47
|
+
Fluent::Test::Driver::Output.new(TestTLSModuleOutput).configure(conf)
|
48
|
+
end
|
49
|
+
|
50
|
+
test 'configure' do
|
51
|
+
assert_equal Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION, driver.instance.ssl_version
|
52
|
+
assert_nil driver.instance.ssl_max_version
|
53
|
+
assert_nil driver.instance.ssl_min_version
|
54
|
+
end
|
55
|
+
|
56
|
+
test 'check USE_TLS_MINMAX_VERSION value' do
|
57
|
+
assert_equal @use_tls_minmax_version, Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
58
|
+
end
|
59
|
+
|
60
|
+
sub_test_case 'set_tls_minmax_version_config' do
|
61
|
+
test 'default' do
|
62
|
+
d = driver('')
|
63
|
+
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
|
64
|
+
if @use_tls_minmax_version
|
65
|
+
if @enabled_tlsv1_3
|
66
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
67
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
68
|
+
else
|
69
|
+
assert_equal({max_version: nil,
|
70
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
71
|
+
|
72
|
+
end
|
73
|
+
else
|
74
|
+
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
test 'errorous cases' do
|
79
|
+
if @use_tls_minmax_version
|
80
|
+
assert_raise(Fluent::ConfigError) do
|
81
|
+
d = driver(%{ssl_max_version TLSv1_2})
|
82
|
+
d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
|
83
|
+
d.instance.ssl_max_version,
|
84
|
+
d.instance.ssl_min_version)
|
85
|
+
end
|
86
|
+
assert_raise(Fluent::ConfigError) do
|
87
|
+
d = driver(%{ssl_min_version TLSv1_2})
|
88
|
+
d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
|
89
|
+
d.instance.ssl_max_version,
|
90
|
+
d.instance.ssl_min_version)
|
91
|
+
end
|
92
|
+
else
|
93
|
+
d1 = driver(%{
|
94
|
+
ssl_max_version TLSv1_2
|
95
|
+
@log_level info
|
96
|
+
})
|
97
|
+
d1.instance.set_tls_minmax_version_config(d1.instance.ssl_version,
|
98
|
+
d1.instance.ssl_max_version,
|
99
|
+
d1.instance.ssl_min_version)
|
100
|
+
|
101
|
+
d1.logs.any? {|a| a.include?("'ssl_max_version' does not have any effect in this environment.") }
|
102
|
+
d2 = driver(%{
|
103
|
+
ssl_min_version TLSv1_2
|
104
|
+
@log_level info
|
105
|
+
})
|
106
|
+
d2.instance.set_tls_minmax_version_config(d2.instance.ssl_version,
|
107
|
+
d2.instance.ssl_max_version,
|
108
|
+
d2.instance.ssl_min_version)
|
109
|
+
d2.logs.any? {|a| a.include?("'ssl_min_version' does not have any effect in this environment.") }
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
test 'min_version & max_version' do
|
114
|
+
config = %{
|
115
|
+
ssl_max_version TLSv1_2
|
116
|
+
ssl_min_version TLSv1_1
|
117
|
+
}
|
118
|
+
d = driver(config)
|
119
|
+
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
|
120
|
+
d.instance.ssl_max_version,
|
121
|
+
d.instance.ssl_min_version)
|
122
|
+
if @use_tls_minmax_version
|
123
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_2_VERSION,
|
124
|
+
min_version: OpenSSL::SSL::TLS1_1_VERSION}, ssl_version_options)
|
125
|
+
else
|
126
|
+
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
test 'TLSv1.3' do
|
131
|
+
omit "openssl gem does not support TLSv1.3" unless @enabled_tlsv1_3
|
132
|
+
config = %{
|
133
|
+
ssl_max_version TLSv1_3
|
134
|
+
ssl_min_version TLSv1_2
|
135
|
+
}
|
136
|
+
d = driver(config)
|
137
|
+
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
|
138
|
+
d.instance.ssl_max_version,
|
139
|
+
d.instance.ssl_min_version)
|
140
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
141
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
142
|
+
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
@@ -0,0 +1,215 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'date'
|
3
|
+
require 'fluent/test/helpers'
|
4
|
+
require 'json'
|
5
|
+
require 'fluent/test/driver/filter'
|
6
|
+
require 'flexmock/test_unit'
|
7
|
+
require 'fluent/plugin/filter_elasticsearch_genid'
|
8
|
+
|
9
|
+
class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
10
|
+
include FlexMock::TestCase
|
11
|
+
include Fluent::Test::Helpers
|
12
|
+
|
13
|
+
def setup
|
14
|
+
Fluent::Test.setup
|
15
|
+
end
|
16
|
+
|
17
|
+
def create_driver(conf='')
|
18
|
+
Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
|
19
|
+
end
|
20
|
+
|
21
|
+
test "invalid configuration" do
|
22
|
+
assert_raise(Fluent::ConfigError) do
|
23
|
+
create_driver("use_record_as_seed true")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def sample_record
|
28
|
+
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_configure
|
32
|
+
d = create_driver
|
33
|
+
assert_equal '_hash', d.instance.hash_id_key
|
34
|
+
end
|
35
|
+
|
36
|
+
data("default" => {"hash_id_key" => "_hash"},
|
37
|
+
"custom_key" => {"hash_id_key" => "_edited"},
|
38
|
+
)
|
39
|
+
def test_filter(data)
|
40
|
+
d = create_driver("hash_id_key #{data["hash_id_key"]}")
|
41
|
+
flexmock(SecureRandom).should_receive(:uuid)
|
42
|
+
.and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
|
43
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
44
|
+
d.run(default_tag: 'test') do
|
45
|
+
d.feed(time, sample_record)
|
46
|
+
end
|
47
|
+
assert_equal(Base64.strict_encode64(SecureRandom.uuid),
|
48
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
49
|
+
end
|
50
|
+
|
51
|
+
class UseRecordAsSeedTest < self
|
52
|
+
data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
|
53
|
+
"sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
|
54
|
+
"sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
|
55
|
+
"sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
|
56
|
+
)
|
57
|
+
def test_simple(data)
|
58
|
+
hash_type, expected = data
|
59
|
+
d = create_driver(%[
|
60
|
+
use_record_as_seed true
|
61
|
+
record_keys age,parent_id,routing_id,custom_key
|
62
|
+
hash_type #{hash_type}
|
63
|
+
])
|
64
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
65
|
+
d.run(default_tag: 'test') do
|
66
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
67
|
+
end
|
68
|
+
assert_equal(expected,
|
69
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
70
|
+
end
|
71
|
+
|
72
|
+
data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
|
73
|
+
"sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
|
74
|
+
"sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
|
75
|
+
"sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
|
76
|
+
def test_record_with_tag(data)
|
77
|
+
hash_type, expected = data
|
78
|
+
d = create_driver(%[
|
79
|
+
use_record_as_seed true
|
80
|
+
record_keys age,parent_id,routing_id,custom_key
|
81
|
+
hash_type #{hash_type}
|
82
|
+
include_tag_in_seed true
|
83
|
+
])
|
84
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
85
|
+
d.run(default_tag: 'test.fluentd') do
|
86
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
87
|
+
end
|
88
|
+
assert_equal(expected,
|
89
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
90
|
+
end
|
91
|
+
|
92
|
+
data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
|
93
|
+
"sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
|
94
|
+
"sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
|
95
|
+
"sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
|
96
|
+
def test_record_with_time(data)
|
97
|
+
hash_type, expected = data
|
98
|
+
d = create_driver(%[
|
99
|
+
use_record_as_seed true
|
100
|
+
record_keys age,parent_id,routing_id,custom_key
|
101
|
+
hash_type #{hash_type}
|
102
|
+
include_time_in_seed true
|
103
|
+
])
|
104
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
105
|
+
d.run(default_tag: 'test.fluentd') do
|
106
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
107
|
+
end
|
108
|
+
assert_equal(expected,
|
109
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
110
|
+
end
|
111
|
+
|
112
|
+
data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
|
113
|
+
"sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
|
114
|
+
"sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
|
115
|
+
"sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
|
116
|
+
def test_record_with_tag_and_time
|
117
|
+
hash_type, expected = data
|
118
|
+
d = create_driver(%[
|
119
|
+
use_record_as_seed true
|
120
|
+
record_keys age,parent_id,routing_id,custom_key
|
121
|
+
hash_type #{hash_type}
|
122
|
+
include_tag_in_seed true
|
123
|
+
include_time_in_seed true
|
124
|
+
])
|
125
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
126
|
+
d.run(default_tag: 'test.fluentd') do
|
127
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
128
|
+
end
|
129
|
+
assert_equal(expected,
|
130
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class UseEntireRecordAsSeedTest < self
|
135
|
+
data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
|
136
|
+
"sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
|
137
|
+
"sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
|
138
|
+
"sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
|
139
|
+
def test_record
|
140
|
+
hash_type, expected = data
|
141
|
+
d = create_driver(%[
|
142
|
+
use_record_as_seed true
|
143
|
+
use_entire_record true
|
144
|
+
hash_type #{hash_type}
|
145
|
+
])
|
146
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
147
|
+
d.run(default_tag: 'test.fluentd') do
|
148
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
149
|
+
end
|
150
|
+
assert_equal(expected,
|
151
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
152
|
+
end
|
153
|
+
|
154
|
+
data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
|
155
|
+
"sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
|
156
|
+
"sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
|
157
|
+
"sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
|
158
|
+
def test_record_with_tag
|
159
|
+
hash_type, expected = data
|
160
|
+
d = create_driver(%[
|
161
|
+
use_record_as_seed true
|
162
|
+
use_entire_record true
|
163
|
+
hash_type #{hash_type}
|
164
|
+
include_tag_in_seed true
|
165
|
+
])
|
166
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
167
|
+
d.run(default_tag: 'test.fluentd') do
|
168
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
169
|
+
end
|
170
|
+
assert_equal(expected,
|
171
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
172
|
+
end
|
173
|
+
|
174
|
+
data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
|
175
|
+
"sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
|
176
|
+
"sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
|
177
|
+
"sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
|
178
|
+
def test_record_with_time
|
179
|
+
hash_type, expected = data
|
180
|
+
d = create_driver(%[
|
181
|
+
use_record_as_seed true
|
182
|
+
use_entire_record true
|
183
|
+
hash_type #{hash_type}
|
184
|
+
include_time_in_seed true
|
185
|
+
])
|
186
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
187
|
+
d.run(default_tag: 'test.fluentd') do
|
188
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
189
|
+
end
|
190
|
+
assert_equal(expected,
|
191
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
192
|
+
end
|
193
|
+
|
194
|
+
data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
|
195
|
+
"sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
|
196
|
+
"sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
|
197
|
+
"sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
|
198
|
+
def test_record_with_tag_and_time
|
199
|
+
hash_type, expected = data
|
200
|
+
d = create_driver(%[
|
201
|
+
use_record_as_seed true
|
202
|
+
use_entire_record true
|
203
|
+
hash_type #{hash_type}
|
204
|
+
include_tag_in_seed true
|
205
|
+
include_time_in_seed true
|
206
|
+
])
|
207
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
208
|
+
d.run(default_tag: 'test.fluentd') do
|
209
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
210
|
+
end
|
211
|
+
assert_equal(expected,
|
212
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|