fluent-plugin-elasticsearch 4.0.6 → 4.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/History.md +25 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +81 -3
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_tls.rb +3 -3
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +67 -32
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +6 -4
- data/test/plugin/test_elasticsearch_tls.rb +2 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +423 -5
- data/test/plugin/test_out_elasticsearch_dynamic.rb +21 -3
- metadata +7 -2
@@ -55,6 +55,7 @@ module Fluent::Plugin
|
|
55
55
|
attr_reader :alias_indexes
|
56
56
|
attr_reader :template_names
|
57
57
|
attr_reader :ssl_version_options
|
58
|
+
attr_reader :compressable_connection
|
58
59
|
|
59
60
|
helpers :event_emitter, :compat_parameters, :record_accessor, :timer
|
60
61
|
|
@@ -89,6 +90,7 @@ EOC
|
|
89
90
|
config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
|
90
91
|
config_param :utc_index, :bool, :default => true
|
91
92
|
config_param :type_name, :string, :default => DEFAULT_TYPE_NAME
|
93
|
+
config_param :suppress_type_name, :bool, :default => false
|
92
94
|
config_param :index_name, :string, :default => "fluentd"
|
93
95
|
config_param :id_key, :string, :default => nil
|
94
96
|
config_param :write_operation, :string, :default => "index"
|
@@ -143,6 +145,7 @@ see: https://github.com/elastic/elasticsearch-ruby/pull/514
|
|
143
145
|
EOC
|
144
146
|
config_param :include_index_in_url, :bool, :default => false
|
145
147
|
config_param :http_backend, :enum, list: [:excon, :typhoeus], :default => :excon
|
148
|
+
config_param :http_backend_excon_nonblock, :bool, :default => true
|
146
149
|
config_param :validate_client_version, :bool, :default => false
|
147
150
|
config_param :prefer_oj_serializer, :bool, :default => false
|
148
151
|
config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "es_rejected_execution_exception"]
|
@@ -158,6 +161,7 @@ EOC
|
|
158
161
|
config_param :enable_ilm, :bool, :default => false
|
159
162
|
config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
|
160
163
|
config_param :ilm_policy, :hash, :default => {}
|
164
|
+
config_param :ilm_policies, :hash, :default => {}
|
161
165
|
config_param :ilm_policy_overwrite, :bool, :default => false
|
162
166
|
config_param :truncate_caches_interval, :time, :default => nil
|
163
167
|
|
@@ -218,6 +222,11 @@ EOC
|
|
218
222
|
log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
|
219
223
|
end
|
220
224
|
|
225
|
+
raise Fluent::ConfigError, "You can't specify ilm_policy and ilm_policies at the same time" unless @ilm_policy.empty? or @ilm_policies.empty?
|
226
|
+
|
227
|
+
unless @ilm_policy.empty?
|
228
|
+
@ilm_policies = { @ilm_policy_id => @ilm_policy }
|
229
|
+
end
|
221
230
|
@alias_indexes = []
|
222
231
|
@template_names = []
|
223
232
|
if !dry_run?
|
@@ -226,14 +235,14 @@ EOC
|
|
226
235
|
raise Fluent::ConfigError, "deflector_alias is prohibited to use with 'logstash_format at same time." if @logstash_format and @deflector_alias
|
227
236
|
end
|
228
237
|
if @ilm_policy.empty? && @ilm_policy_overwrite
|
229
|
-
raise Fluent::ConfigError, "ilm_policy_overwrite
|
238
|
+
raise Fluent::ConfigError, "ilm_policy_overwrite requires a non empty ilm_policy."
|
230
239
|
end
|
231
240
|
if @logstash_format || placeholder_substitution_needed_for_template?
|
232
241
|
class << self
|
233
242
|
alias_method :template_installation, :template_installation_actual
|
234
243
|
end
|
235
244
|
else
|
236
|
-
template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
|
245
|
+
template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
|
237
246
|
end
|
238
247
|
verify_ilm_working if @enable_ilm
|
239
248
|
elsif @templates
|
@@ -296,16 +305,20 @@ EOC
|
|
296
305
|
else
|
297
306
|
@default_elasticsearch_version
|
298
307
|
end
|
299
|
-
if @
|
300
|
-
log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
|
301
|
-
end
|
302
|
-
if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
303
|
-
log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
|
304
|
-
@type_name = '_doc'.freeze
|
305
|
-
end
|
306
|
-
if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
307
|
-
log.info "Detected ES 8.x or above: This parameter has no effect."
|
308
|
+
if @suppress_type_name && @last_seen_major_version >= 7
|
308
309
|
@type_name = nil
|
310
|
+
else
|
311
|
+
if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
312
|
+
log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
|
313
|
+
end
|
314
|
+
if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
315
|
+
log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
|
316
|
+
@type_name = '_doc'.freeze
|
317
|
+
end
|
318
|
+
if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
319
|
+
log.info "Detected ES 8.x or above: This parameter has no effect."
|
320
|
+
@type_name = nil
|
321
|
+
end
|
309
322
|
end
|
310
323
|
|
311
324
|
if @validate_client_version && !dry_run?
|
@@ -327,6 +340,13 @@ EOC
|
|
327
340
|
end
|
328
341
|
end
|
329
342
|
|
343
|
+
if @ssl_version && @scheme == :https
|
344
|
+
if !@http_backend_excon_nonblock
|
345
|
+
log.warn "TLS handshake will be stucked with block connection.
|
346
|
+
Consider to set `http_backend_excon_nonblock` as true"
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
330
350
|
# Consider missing the prefix of "$." in nested key specifiers.
|
331
351
|
@id_key = convert_compat_id_key(@id_key) if @id_key
|
332
352
|
@parent_key = convert_compat_id_key(@parent_key) if @parent_key
|
@@ -335,6 +355,7 @@ EOC
|
|
335
355
|
@routing_key_name = configure_routing_key_name
|
336
356
|
@meta_config_map = create_meta_config_map
|
337
357
|
@current_config = nil
|
358
|
+
@compressable_connection = false
|
338
359
|
|
339
360
|
@ignore_exception_classes = @ignore_exceptions.map do |exception|
|
340
361
|
unless Object.const_defined?(exception)
|
@@ -409,7 +430,7 @@ EOC
|
|
409
430
|
def backend_options
|
410
431
|
case @http_backend
|
411
432
|
when :excon
|
412
|
-
{ client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
|
433
|
+
{ client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
|
413
434
|
when :typhoeus
|
414
435
|
require 'typhoeus'
|
415
436
|
{ sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
|
@@ -503,13 +524,15 @@ EOC
|
|
503
524
|
return Time.at(event_time).to_datetime
|
504
525
|
end
|
505
526
|
|
506
|
-
def client(host = nil)
|
527
|
+
def client(host = nil, compress_connection = false)
|
507
528
|
# check here to see if we already have a client connection for the given host
|
508
529
|
connection_options = get_connection_options(host)
|
509
530
|
|
510
531
|
@_es = nil unless is_existing_connection(connection_options[:hosts])
|
532
|
+
@_es = nil unless @compressable_connection == compress_connection
|
511
533
|
|
512
534
|
@_es ||= begin
|
535
|
+
@compressable_connection = compress_connection
|
513
536
|
@current_config = connection_options[:hosts].clone
|
514
537
|
adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
|
515
538
|
local_reload_connections = @reload_connections
|
@@ -517,7 +540,7 @@ EOC
|
|
517
540
|
local_reload_connections = @reload_after
|
518
541
|
end
|
519
542
|
|
520
|
-
gzip_headers = if
|
543
|
+
gzip_headers = if compress_connection
|
521
544
|
{'Content-Encoding' => 'gzip'}
|
522
545
|
else
|
523
546
|
{}
|
@@ -543,7 +566,7 @@ EOC
|
|
543
566
|
},
|
544
567
|
sniffer_class: @sniffer_class,
|
545
568
|
serializer_class: @serializer_class,
|
546
|
-
compression:
|
569
|
+
compression: compress_connection,
|
547
570
|
}), &adapter_conf)
|
548
571
|
Elasticsearch::Client.new transport: transport
|
549
572
|
end
|
@@ -707,7 +730,12 @@ EOC
|
|
707
730
|
else
|
708
731
|
pipeline = nil
|
709
732
|
end
|
710
|
-
|
733
|
+
if @ilm_policy_id
|
734
|
+
ilm_policy_id = extract_placeholders(@ilm_policy_id, chunk)
|
735
|
+
else
|
736
|
+
ilm_policy_id = nil
|
737
|
+
end
|
738
|
+
return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline, ilm_policy_id
|
711
739
|
end
|
712
740
|
|
713
741
|
def multi_workers_ready?
|
@@ -733,9 +761,9 @@ EOC
|
|
733
761
|
begin
|
734
762
|
meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
|
735
763
|
info = if @include_index_in_url
|
736
|
-
RequestInfo.new(host, meta.delete("_index".freeze), meta
|
764
|
+
RequestInfo.new(host, meta.delete("_index".freeze), meta.delete("_alias".freeze))
|
737
765
|
else
|
738
|
-
RequestInfo.new(host, nil, meta
|
766
|
+
RequestInfo.new(host, nil, meta.delete("_alias".freeze))
|
739
767
|
end
|
740
768
|
|
741
769
|
if split_request?(bulk_message, info)
|
@@ -781,7 +809,7 @@ EOC
|
|
781
809
|
end
|
782
810
|
|
783
811
|
def process_message(tag, meta, header, time, record, extracted_values)
|
784
|
-
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias,
|
812
|
+
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
|
785
813
|
|
786
814
|
if @flatten_hashes
|
787
815
|
record = flatten_record(record)
|
@@ -804,17 +832,19 @@ EOC
|
|
804
832
|
|
805
833
|
target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
|
806
834
|
if target_index_parent && target_index_parent[target_index_child_key]
|
807
|
-
target_index = target_index_parent.delete(target_index_child_key)
|
835
|
+
target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
|
808
836
|
elsif @logstash_format
|
809
837
|
dt = dt.new_offset(0) if @utc_index
|
810
838
|
target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
|
839
|
+
target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
|
811
840
|
else
|
812
|
-
target_index = index_name
|
841
|
+
target_index_alias = target_index = index_name
|
813
842
|
end
|
814
843
|
|
815
844
|
# Change target_index to lower-case since Elasticsearch doesn't
|
816
845
|
# allow upper-case characters in index names.
|
817
846
|
target_index = target_index.downcase
|
847
|
+
target_index_alias = target_index_alias.downcase
|
818
848
|
if @include_tag_key
|
819
849
|
record[@tag_key] = tag
|
820
850
|
end
|
@@ -833,7 +863,9 @@ EOC
|
|
833
863
|
target_type = nil
|
834
864
|
end
|
835
865
|
else
|
836
|
-
if @
|
866
|
+
if @suppress_type_name && @last_seen_major_version >= 7
|
867
|
+
target_type = nil
|
868
|
+
elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
837
869
|
log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
|
838
870
|
target_type = '_doc'.freeze
|
839
871
|
elsif @last_seen_major_version >= 8
|
@@ -847,6 +879,7 @@ EOC
|
|
847
879
|
meta.clear
|
848
880
|
meta["_index".freeze] = target_index
|
849
881
|
meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
|
882
|
+
meta["_alias".freeze] = target_index_alias
|
850
883
|
|
851
884
|
if @pipeline
|
852
885
|
meta["pipeline".freeze] = pipeline
|
@@ -889,16 +922,17 @@ EOC
|
|
889
922
|
placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
|
890
923
|
placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
|
891
924
|
placeholder?(:deflector_alias, @deflector_alias.to_s) ||
|
892
|
-
placeholder?(:application_name, @application_name.to_s)
|
925
|
+
placeholder?(:application_name, @application_name.to_s) ||
|
926
|
+
placeholder?(:ilm_policy_id, @ilm_policy_id.to_s)
|
893
927
|
log.debug("Need substitution: #{need_substitution}")
|
894
928
|
need_substitution
|
895
929
|
end
|
896
930
|
|
897
|
-
def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
|
931
|
+
def template_installation(deflector_alias, template_name, customize_template, application_name, ilm_policy_id, target_index, host)
|
898
932
|
# for safety.
|
899
933
|
end
|
900
934
|
|
901
|
-
def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
|
935
|
+
def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
|
902
936
|
if template_name && @template_file
|
903
937
|
if @alias_indexes.include? deflector_alias
|
904
938
|
log.debug("Index alias #{deflector_alias} already exists (cached)")
|
@@ -907,11 +941,12 @@ EOC
|
|
907
941
|
else
|
908
942
|
retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
|
909
943
|
if customize_template
|
910
|
-
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias,
|
944
|
+
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
|
911
945
|
else
|
912
|
-
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias,
|
946
|
+
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
|
913
947
|
end
|
914
|
-
|
948
|
+
ilm_policy = @ilm_policies[ilm_policy_id] || {}
|
949
|
+
create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
|
915
950
|
end
|
916
951
|
@alias_indexes << deflector_alias unless deflector_alias.nil?
|
917
952
|
@template_names << template_name unless template_name.nil?
|
@@ -922,11 +957,11 @@ EOC
|
|
922
957
|
# send_bulk given a specific bulk request, the original tag,
|
923
958
|
# chunk, and bulk_message_count
|
924
959
|
def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
|
925
|
-
logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
|
960
|
+
logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
|
926
961
|
if deflector_alias
|
927
|
-
template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
|
962
|
+
template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
|
928
963
|
else
|
929
|
-
template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
|
964
|
+
template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
|
930
965
|
end
|
931
966
|
|
932
967
|
begin
|
@@ -939,7 +974,7 @@ EOC
|
|
939
974
|
data
|
940
975
|
end
|
941
976
|
|
942
|
-
response = client(info.host).bulk body: prepared_data, index: info.index
|
977
|
+
response = client(info.host, compression).bulk body: prepared_data, index: info.index
|
943
978
|
log.on_trace { log.trace "bulk response: #{response}" }
|
944
979
|
|
945
980
|
if response['errors']
|
@@ -35,16 +35,18 @@ module Fluent::Plugin
|
|
35
35
|
end
|
36
36
|
|
37
37
|
|
38
|
-
def client(host = nil)
|
38
|
+
def client(host = nil, compress_connection = false)
|
39
39
|
# check here to see if we already have a client connection for the given host
|
40
40
|
connection_options = get_connection_options(host)
|
41
41
|
|
42
42
|
@_es = nil unless is_existing_connection(connection_options[:hosts])
|
43
|
+
@_es = nil unless @compressable_connection == compress_connection
|
43
44
|
|
44
45
|
@_es ||= begin
|
46
|
+
@compressable_connection = compress_connection
|
45
47
|
@current_config = connection_options[:hosts].clone
|
46
48
|
adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
|
47
|
-
gzip_headers = if
|
49
|
+
gzip_headers = if compress_connection
|
48
50
|
{'Content-Encoding' => 'gzip'}
|
49
51
|
else
|
50
52
|
{}
|
@@ -67,7 +69,7 @@ module Fluent::Plugin
|
|
67
69
|
password: @password,
|
68
70
|
scheme: @scheme
|
69
71
|
},
|
70
|
-
compression:
|
72
|
+
compression: compress_connection,
|
71
73
|
}), &adapter_conf)
|
72
74
|
Elasticsearch::Client.new transport: transport
|
73
75
|
end
|
@@ -228,7 +230,7 @@ module Fluent::Plugin
|
|
228
230
|
else
|
229
231
|
data
|
230
232
|
end
|
231
|
-
response = client(host).bulk body: prepared_data, index: index
|
233
|
+
response = client(host, compression).bulk body: prepared_data, index: index
|
232
234
|
if response['errors']
|
233
235
|
log.error "Could not push log to Elasticsearch: #{response}"
|
234
236
|
end
|
@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
|
|
62
62
|
d = driver('')
|
63
63
|
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
|
64
64
|
if @use_tls_minmax_version
|
65
|
-
assert_equal({max_version: OpenSSL::SSL::
|
66
|
-
min_version: OpenSSL::SSL::
|
65
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
66
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
67
67
|
else
|
68
68
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
69
69
|
end
|
@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
18
18
|
Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
|
19
19
|
end
|
20
20
|
|
21
|
+
test "invalid configuration" do
|
22
|
+
assert_raise(Fluent::ConfigError) do
|
23
|
+
create_driver("use_record_as_seed true")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
21
27
|
def sample_record
|
22
28
|
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
23
29
|
end
|
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
41
47
|
assert_equal(Base64.strict_encode64(SecureRandom.uuid),
|
42
48
|
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
43
49
|
end
|
50
|
+
|
51
|
+
class UseRecordAsSeedTest < self
|
52
|
+
data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
|
53
|
+
"sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
|
54
|
+
"sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
|
55
|
+
"sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
|
56
|
+
)
|
57
|
+
def test_simple(data)
|
58
|
+
hash_type, expected = data
|
59
|
+
d = create_driver(%[
|
60
|
+
use_record_as_seed true
|
61
|
+
record_keys age,parent_id,routing_id,custom_key
|
62
|
+
hash_type #{hash_type}
|
63
|
+
])
|
64
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
65
|
+
d.run(default_tag: 'test') do
|
66
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
67
|
+
end
|
68
|
+
assert_equal(expected,
|
69
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
70
|
+
end
|
71
|
+
|
72
|
+
data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
|
73
|
+
"sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
|
74
|
+
"sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
|
75
|
+
"sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
|
76
|
+
def test_record_with_tag(data)
|
77
|
+
hash_type, expected = data
|
78
|
+
d = create_driver(%[
|
79
|
+
use_record_as_seed true
|
80
|
+
record_keys age,parent_id,routing_id,custom_key
|
81
|
+
hash_type #{hash_type}
|
82
|
+
include_tag_in_seed true
|
83
|
+
])
|
84
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
85
|
+
d.run(default_tag: 'test.fluentd') do
|
86
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
87
|
+
end
|
88
|
+
assert_equal(expected,
|
89
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
90
|
+
end
|
91
|
+
|
92
|
+
data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
|
93
|
+
"sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
|
94
|
+
"sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
|
95
|
+
"sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
|
96
|
+
def test_record_with_time(data)
|
97
|
+
hash_type, expected = data
|
98
|
+
d = create_driver(%[
|
99
|
+
use_record_as_seed true
|
100
|
+
record_keys age,parent_id,routing_id,custom_key
|
101
|
+
hash_type #{hash_type}
|
102
|
+
include_time_in_seed true
|
103
|
+
])
|
104
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
105
|
+
d.run(default_tag: 'test.fluentd') do
|
106
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
107
|
+
end
|
108
|
+
assert_equal(expected,
|
109
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
110
|
+
end
|
111
|
+
|
112
|
+
data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
|
113
|
+
"sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
|
114
|
+
"sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
|
115
|
+
"sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
|
116
|
+
def test_record_with_tag_and_time
|
117
|
+
hash_type, expected = data
|
118
|
+
d = create_driver(%[
|
119
|
+
use_record_as_seed true
|
120
|
+
record_keys age,parent_id,routing_id,custom_key
|
121
|
+
hash_type #{hash_type}
|
122
|
+
include_tag_in_seed true
|
123
|
+
include_time_in_seed true
|
124
|
+
])
|
125
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
126
|
+
d.run(default_tag: 'test.fluentd') do
|
127
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
128
|
+
end
|
129
|
+
assert_equal(expected,
|
130
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class UseEntireRecordAsSeedTest < self
|
135
|
+
data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
|
136
|
+
"sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
|
137
|
+
"sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
|
138
|
+
"sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
|
139
|
+
def test_record
|
140
|
+
hash_type, expected = data
|
141
|
+
d = create_driver(%[
|
142
|
+
use_record_as_seed true
|
143
|
+
use_entire_record true
|
144
|
+
hash_type #{hash_type}
|
145
|
+
])
|
146
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
147
|
+
d.run(default_tag: 'test.fluentd') do
|
148
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
149
|
+
end
|
150
|
+
assert_equal(expected,
|
151
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
152
|
+
end
|
153
|
+
|
154
|
+
data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
|
155
|
+
"sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
|
156
|
+
"sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
|
157
|
+
"sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
|
158
|
+
def test_record_with_tag
|
159
|
+
hash_type, expected = data
|
160
|
+
d = create_driver(%[
|
161
|
+
use_record_as_seed true
|
162
|
+
use_entire_record true
|
163
|
+
hash_type #{hash_type}
|
164
|
+
include_tag_in_seed true
|
165
|
+
])
|
166
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
167
|
+
d.run(default_tag: 'test.fluentd') do
|
168
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
169
|
+
end
|
170
|
+
assert_equal(expected,
|
171
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
172
|
+
end
|
173
|
+
|
174
|
+
data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
|
175
|
+
"sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
|
176
|
+
"sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
|
177
|
+
"sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
|
178
|
+
def test_record_with_time
|
179
|
+
hash_type, expected = data
|
180
|
+
d = create_driver(%[
|
181
|
+
use_record_as_seed true
|
182
|
+
use_entire_record true
|
183
|
+
hash_type #{hash_type}
|
184
|
+
include_time_in_seed true
|
185
|
+
])
|
186
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
187
|
+
d.run(default_tag: 'test.fluentd') do
|
188
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
189
|
+
end
|
190
|
+
assert_equal(expected,
|
191
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
192
|
+
end
|
193
|
+
|
194
|
+
data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
|
195
|
+
"sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
|
196
|
+
"sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
|
197
|
+
"sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
|
198
|
+
def test_record_with_tag_and_time
|
199
|
+
hash_type, expected = data
|
200
|
+
d = create_driver(%[
|
201
|
+
use_record_as_seed true
|
202
|
+
use_entire_record true
|
203
|
+
hash_type #{hash_type}
|
204
|
+
include_tag_in_seed true
|
205
|
+
include_time_in_seed true
|
206
|
+
])
|
207
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
208
|
+
d.run(default_tag: 'test.fluentd') do
|
209
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
210
|
+
end
|
211
|
+
assert_equal(expected,
|
212
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
213
|
+
end
|
214
|
+
end
|
44
215
|
end
|