fluent-plugin-elasticsearch 4.0.9 → 4.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/History.md +22 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +74 -9
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +9 -0
- data/lib/fluent/plugin/elasticsearch_index_template.rb +19 -11
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +48 -25
- data/test/plugin/test_elasticsearch_fallback_selector.rb +73 -0
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +171 -80
- metadata +10 -5
@@ -25,6 +25,7 @@ require_relative 'elasticsearch_error_handler'
|
|
25
25
|
require_relative 'elasticsearch_index_template'
|
26
26
|
require_relative 'elasticsearch_index_lifecycle_management'
|
27
27
|
require_relative 'elasticsearch_tls'
|
28
|
+
require_relative 'elasticsearch_fallback_selector'
|
28
29
|
begin
|
29
30
|
require_relative 'oj_serializer'
|
30
31
|
rescue LoadError
|
@@ -50,7 +51,7 @@ module Fluent::Plugin
|
|
50
51
|
end
|
51
52
|
end
|
52
53
|
|
53
|
-
RequestInfo = Struct.new(:host, :index, :ilm_index)
|
54
|
+
RequestInfo = Struct.new(:host, :index, :ilm_index, :ilm_alias)
|
54
55
|
|
55
56
|
attr_reader :alias_indexes
|
56
57
|
attr_reader :template_names
|
@@ -90,6 +91,7 @@ EOC
|
|
90
91
|
config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
|
91
92
|
config_param :utc_index, :bool, :default => true
|
92
93
|
config_param :type_name, :string, :default => DEFAULT_TYPE_NAME
|
94
|
+
config_param :suppress_type_name, :bool, :default => false
|
93
95
|
config_param :index_name, :string, :default => "fluentd"
|
94
96
|
config_param :id_key, :string, :default => nil
|
95
97
|
config_param :write_operation, :string, :default => "index"
|
@@ -136,6 +138,7 @@ EOC
|
|
136
138
|
config_param :with_transporter_log, :bool, :default => false
|
137
139
|
config_param :emit_error_for_missing_id, :bool, :default => false
|
138
140
|
config_param :sniffer_class_name, :string, :default => nil
|
141
|
+
config_param :selector_class_name, :string, :default => nil
|
139
142
|
config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
|
140
143
|
config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson"], :default => :"application/json",
|
141
144
|
:deprecated => <<EOC
|
@@ -231,10 +234,10 @@ EOC
|
|
231
234
|
if !dry_run?
|
232
235
|
if @template_name && @template_file
|
233
236
|
if @enable_ilm
|
234
|
-
raise Fluent::ConfigError, "deflector_alias is prohibited to use with
|
237
|
+
raise Fluent::ConfigError, "deflector_alias is prohibited to use with enable_ilm at same time." if @deflector_alias
|
235
238
|
end
|
236
239
|
if @ilm_policy.empty? && @ilm_policy_overwrite
|
237
|
-
raise Fluent::ConfigError, "ilm_policy_overwrite
|
240
|
+
raise Fluent::ConfigError, "ilm_policy_overwrite requires a non empty ilm_policy."
|
238
241
|
end
|
239
242
|
if @logstash_format || placeholder_substitution_needed_for_template?
|
240
243
|
class << self
|
@@ -299,21 +302,32 @@ EOC
|
|
299
302
|
raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
|
300
303
|
end
|
301
304
|
|
305
|
+
@selector_class = nil
|
306
|
+
begin
|
307
|
+
@selector_class = Object.const_get(@selector_class_name) if @selector_class_name
|
308
|
+
rescue Exception => ex
|
309
|
+
raise Fluent::ConfigError, "Could not load selector class #{@selector_class_name}: #{ex}"
|
310
|
+
end
|
311
|
+
|
302
312
|
@last_seen_major_version = if major_version = handle_last_seen_es_major_version
|
303
313
|
major_version
|
304
314
|
else
|
305
315
|
@default_elasticsearch_version
|
306
316
|
end
|
307
|
-
if @
|
308
|
-
log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
|
309
|
-
end
|
310
|
-
if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
311
|
-
log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
|
312
|
-
@type_name = '_doc'.freeze
|
313
|
-
end
|
314
|
-
if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
315
|
-
log.info "Detected ES 8.x or above: This parameter has no effect."
|
317
|
+
if @suppress_type_name && @last_seen_major_version >= 7
|
316
318
|
@type_name = nil
|
319
|
+
else
|
320
|
+
if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
321
|
+
log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
|
322
|
+
end
|
323
|
+
if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
324
|
+
log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
|
325
|
+
@type_name = '_doc'.freeze
|
326
|
+
end
|
327
|
+
if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
328
|
+
log.info "Detected ES 8.x or above: This parameter has no effect."
|
329
|
+
@type_name = nil
|
330
|
+
end
|
317
331
|
end
|
318
332
|
|
319
333
|
if @validate_client_version && !dry_run?
|
@@ -447,7 +461,10 @@ EOC
|
|
447
461
|
|
448
462
|
def detect_es_major_version
|
449
463
|
@_es_info ||= client.info
|
450
|
-
@_es_info
|
464
|
+
unless version = @_es_info.dig("version", "number")
|
465
|
+
version = @default_elasticsearch_version
|
466
|
+
end
|
467
|
+
version.to_i
|
451
468
|
end
|
452
469
|
|
453
470
|
def client_library_version
|
@@ -561,6 +578,7 @@ EOC
|
|
561
578
|
},
|
562
579
|
sniffer_class: @sniffer_class,
|
563
580
|
serializer_class: @serializer_class,
|
581
|
+
selector_class: @selector_class,
|
564
582
|
compression: compress_connection,
|
565
583
|
}), &adapter_conf)
|
566
584
|
Elasticsearch::Client.new transport: transport
|
@@ -756,9 +774,9 @@ EOC
|
|
756
774
|
begin
|
757
775
|
meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
|
758
776
|
info = if @include_index_in_url
|
759
|
-
RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze])
|
777
|
+
RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
|
760
778
|
else
|
761
|
-
RequestInfo.new(host, nil, meta["_index".freeze])
|
779
|
+
RequestInfo.new(host, nil, meta["_index".freeze], meta.delete("_alias".freeze))
|
762
780
|
end
|
763
781
|
|
764
782
|
if split_request?(bulk_message, info)
|
@@ -804,7 +822,7 @@ EOC
|
|
804
822
|
end
|
805
823
|
|
806
824
|
def process_message(tag, meta, header, time, record, extracted_values)
|
807
|
-
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias,
|
825
|
+
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
|
808
826
|
|
809
827
|
if @flatten_hashes
|
810
828
|
record = flatten_record(record)
|
@@ -827,17 +845,19 @@ EOC
|
|
827
845
|
|
828
846
|
target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
|
829
847
|
if target_index_parent && target_index_parent[target_index_child_key]
|
830
|
-
target_index = target_index_parent.delete(target_index_child_key)
|
848
|
+
target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
|
831
849
|
elsif @logstash_format
|
832
850
|
dt = dt.new_offset(0) if @utc_index
|
833
851
|
target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
|
852
|
+
target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
|
834
853
|
else
|
835
|
-
target_index = index_name
|
854
|
+
target_index_alias = target_index = index_name
|
836
855
|
end
|
837
856
|
|
838
857
|
# Change target_index to lower-case since Elasticsearch doesn't
|
839
858
|
# allow upper-case characters in index names.
|
840
859
|
target_index = target_index.downcase
|
860
|
+
target_index_alias = target_index_alias.downcase
|
841
861
|
if @include_tag_key
|
842
862
|
record[@tag_key] = tag
|
843
863
|
end
|
@@ -856,7 +876,9 @@ EOC
|
|
856
876
|
target_type = nil
|
857
877
|
end
|
858
878
|
else
|
859
|
-
if @
|
879
|
+
if @suppress_type_name && @last_seen_major_version >= 7
|
880
|
+
target_type = nil
|
881
|
+
elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
860
882
|
log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
|
861
883
|
target_type = '_doc'.freeze
|
862
884
|
elsif @last_seen_major_version >= 8
|
@@ -870,6 +892,7 @@ EOC
|
|
870
892
|
meta.clear
|
871
893
|
meta["_index".freeze] = target_index
|
872
894
|
meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
|
895
|
+
meta["_alias".freeze] = target_index_alias
|
873
896
|
|
874
897
|
if @pipeline
|
875
898
|
meta["pipeline".freeze] = pipeline
|
@@ -924,16 +947,16 @@ EOC
|
|
924
947
|
|
925
948
|
def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
|
926
949
|
if template_name && @template_file
|
927
|
-
if @alias_indexes.include?
|
950
|
+
if !@logstash_format && @alias_indexes.include?(deflector_alias)
|
928
951
|
log.debug("Index alias #{deflector_alias} already exists (cached)")
|
929
|
-
elsif @template_names.include?
|
952
|
+
elsif !@logstash_format && @template_names.include?(template_name)
|
930
953
|
log.debug("Template name #{template_name} already exists (cached)")
|
931
954
|
else
|
932
955
|
retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
|
933
956
|
if customize_template
|
934
|
-
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
|
957
|
+
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
|
935
958
|
else
|
936
|
-
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
|
959
|
+
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
|
937
960
|
end
|
938
961
|
ilm_policy = @ilm_policies[ilm_policy_id] || {}
|
939
962
|
create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
|
@@ -947,11 +970,11 @@ EOC
|
|
947
970
|
# send_bulk given a specific bulk request, the original tag,
|
948
971
|
# chunk, and bulk_message_count
|
949
972
|
def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
|
950
|
-
|
973
|
+
_logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
|
951
974
|
if deflector_alias
|
952
975
|
template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
|
953
976
|
else
|
954
|
-
template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ?
|
977
|
+
template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? info.ilm_alias : index_name, ilm_policy_id, info.host)
|
955
978
|
end
|
956
979
|
|
957
980
|
begin
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/output'
|
3
|
+
require 'fluent/plugin/out_elasticsearch'
|
4
|
+
|
5
|
+
class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
6
|
+
attr_accessor :index_cmds
|
7
|
+
|
8
|
+
def setup
|
9
|
+
Fluent::Test.setup
|
10
|
+
@driver = nil
|
11
|
+
log = Fluent::Engine.log
|
12
|
+
log.out.logs.slice!(0, log.out.logs.length)
|
13
|
+
end
|
14
|
+
|
15
|
+
def stub_elastic(url="http://localhost:9200/_bulk")
|
16
|
+
stub_request(:post, url).with do |req|
|
17
|
+
@index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
|
22
|
+
body ="{\"version\":{\"number\":\"#{version}\"}}"
|
23
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
24
|
+
end
|
25
|
+
|
26
|
+
def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
|
27
|
+
stub_request(:get, url).to_return(:status => [404, "Not Found"])
|
28
|
+
end
|
29
|
+
|
30
|
+
def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
|
31
|
+
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
|
32
|
+
end
|
33
|
+
|
34
|
+
def sample_record(content={})
|
35
|
+
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
|
36
|
+
end
|
37
|
+
|
38
|
+
def driver(conf='')
|
39
|
+
@driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
|
40
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
41
|
+
if !defined?(Fluent::Plugin::Output)
|
42
|
+
def format(tag, time, record)
|
43
|
+
[time, record].to_msgpack
|
44
|
+
end
|
45
|
+
end
|
46
|
+
}.configure(conf)
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_fallback_on_info
|
50
|
+
stub_elastic_info_not_found("http://localhost:9202/")
|
51
|
+
stub_elastic_info_unavailable("http://localhost:9201/")
|
52
|
+
stub_elastic_info
|
53
|
+
stub_elastic
|
54
|
+
config = %[
|
55
|
+
hosts localhost:9202,localhost:9201,localhost:9200
|
56
|
+
selector_class_name Fluent::Plugin::ElasticseatchFallbackSelector
|
57
|
+
@log_level debug
|
58
|
+
with_transporter_log true
|
59
|
+
reload_connections true
|
60
|
+
reload_after 10
|
61
|
+
]
|
62
|
+
assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
|
63
|
+
driver(config)
|
64
|
+
end
|
65
|
+
driver.run(default_tag: 'test') do
|
66
|
+
driver.feed(sample_record)
|
67
|
+
end
|
68
|
+
assert_equal(2, index_cmds.length)
|
69
|
+
assert_equal("fluentd", index_cmds.first['index']['_index'])
|
70
|
+
end
|
71
|
+
|
72
|
+
# TODO: on feed phase test case
|
73
|
+
end
|
@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
18
18
|
Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
|
19
19
|
end
|
20
20
|
|
21
|
+
test "invalid configuration" do
|
22
|
+
assert_raise(Fluent::ConfigError) do
|
23
|
+
create_driver("use_record_as_seed true")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
21
27
|
def sample_record
|
22
28
|
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
23
29
|
end
|
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
41
47
|
assert_equal(Base64.strict_encode64(SecureRandom.uuid),
|
42
48
|
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
43
49
|
end
|
50
|
+
|
51
|
+
class UseRecordAsSeedTest < self
|
52
|
+
data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
|
53
|
+
"sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
|
54
|
+
"sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
|
55
|
+
"sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
|
56
|
+
)
|
57
|
+
def test_simple(data)
|
58
|
+
hash_type, expected = data
|
59
|
+
d = create_driver(%[
|
60
|
+
use_record_as_seed true
|
61
|
+
record_keys age,parent_id,routing_id,custom_key
|
62
|
+
hash_type #{hash_type}
|
63
|
+
])
|
64
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
65
|
+
d.run(default_tag: 'test') do
|
66
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
67
|
+
end
|
68
|
+
assert_equal(expected,
|
69
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
70
|
+
end
|
71
|
+
|
72
|
+
data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
|
73
|
+
"sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
|
74
|
+
"sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
|
75
|
+
"sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
|
76
|
+
def test_record_with_tag(data)
|
77
|
+
hash_type, expected = data
|
78
|
+
d = create_driver(%[
|
79
|
+
use_record_as_seed true
|
80
|
+
record_keys age,parent_id,routing_id,custom_key
|
81
|
+
hash_type #{hash_type}
|
82
|
+
include_tag_in_seed true
|
83
|
+
])
|
84
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
85
|
+
d.run(default_tag: 'test.fluentd') do
|
86
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
87
|
+
end
|
88
|
+
assert_equal(expected,
|
89
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
90
|
+
end
|
91
|
+
|
92
|
+
data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
|
93
|
+
"sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
|
94
|
+
"sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
|
95
|
+
"sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
|
96
|
+
def test_record_with_time(data)
|
97
|
+
hash_type, expected = data
|
98
|
+
d = create_driver(%[
|
99
|
+
use_record_as_seed true
|
100
|
+
record_keys age,parent_id,routing_id,custom_key
|
101
|
+
hash_type #{hash_type}
|
102
|
+
include_time_in_seed true
|
103
|
+
])
|
104
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
105
|
+
d.run(default_tag: 'test.fluentd') do
|
106
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
107
|
+
end
|
108
|
+
assert_equal(expected,
|
109
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
110
|
+
end
|
111
|
+
|
112
|
+
data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
|
113
|
+
"sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
|
114
|
+
"sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
|
115
|
+
"sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
|
116
|
+
def test_record_with_tag_and_time
|
117
|
+
hash_type, expected = data
|
118
|
+
d = create_driver(%[
|
119
|
+
use_record_as_seed true
|
120
|
+
record_keys age,parent_id,routing_id,custom_key
|
121
|
+
hash_type #{hash_type}
|
122
|
+
include_tag_in_seed true
|
123
|
+
include_time_in_seed true
|
124
|
+
])
|
125
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
126
|
+
d.run(default_tag: 'test.fluentd') do
|
127
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
128
|
+
end
|
129
|
+
assert_equal(expected,
|
130
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class UseEntireRecordAsSeedTest < self
|
135
|
+
data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
|
136
|
+
"sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
|
137
|
+
"sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
|
138
|
+
"sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
|
139
|
+
def test_record
|
140
|
+
hash_type, expected = data
|
141
|
+
d = create_driver(%[
|
142
|
+
use_record_as_seed true
|
143
|
+
use_entire_record true
|
144
|
+
hash_type #{hash_type}
|
145
|
+
])
|
146
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
147
|
+
d.run(default_tag: 'test.fluentd') do
|
148
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
149
|
+
end
|
150
|
+
assert_equal(expected,
|
151
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
152
|
+
end
|
153
|
+
|
154
|
+
data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
|
155
|
+
"sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
|
156
|
+
"sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
|
157
|
+
"sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
|
158
|
+
def test_record_with_tag
|
159
|
+
hash_type, expected = data
|
160
|
+
d = create_driver(%[
|
161
|
+
use_record_as_seed true
|
162
|
+
use_entire_record true
|
163
|
+
hash_type #{hash_type}
|
164
|
+
include_tag_in_seed true
|
165
|
+
])
|
166
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
167
|
+
d.run(default_tag: 'test.fluentd') do
|
168
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
169
|
+
end
|
170
|
+
assert_equal(expected,
|
171
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
172
|
+
end
|
173
|
+
|
174
|
+
data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
|
175
|
+
"sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
|
176
|
+
"sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
|
177
|
+
"sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
|
178
|
+
def test_record_with_time
|
179
|
+
hash_type, expected = data
|
180
|
+
d = create_driver(%[
|
181
|
+
use_record_as_seed true
|
182
|
+
use_entire_record true
|
183
|
+
hash_type #{hash_type}
|
184
|
+
include_time_in_seed true
|
185
|
+
])
|
186
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
187
|
+
d.run(default_tag: 'test.fluentd') do
|
188
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
189
|
+
end
|
190
|
+
assert_equal(expected,
|
191
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
192
|
+
end
|
193
|
+
|
194
|
+
data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
|
195
|
+
"sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
|
196
|
+
"sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
|
197
|
+
"sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
|
198
|
+
def test_record_with_tag_and_time
|
199
|
+
hash_type, expected = data
|
200
|
+
d = create_driver(%[
|
201
|
+
use_record_as_seed true
|
202
|
+
use_entire_record true
|
203
|
+
hash_type #{hash_type}
|
204
|
+
include_tag_in_seed true
|
205
|
+
include_time_in_seed true
|
206
|
+
])
|
207
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
208
|
+
d.run(default_tag: 'test.fluentd') do
|
209
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
210
|
+
end
|
211
|
+
assert_equal(expected,
|
212
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
213
|
+
end
|
214
|
+
end
|
44
215
|
end
|