fluent-plugin-elasticsearch 4.0.6 → 4.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -55,6 +55,7 @@ module Fluent::Plugin
55
55
  attr_reader :alias_indexes
56
56
  attr_reader :template_names
57
57
  attr_reader :ssl_version_options
58
+ attr_reader :compressable_connection
58
59
 
59
60
  helpers :event_emitter, :compat_parameters, :record_accessor, :timer
60
61
 
@@ -89,6 +90,7 @@ EOC
89
90
  config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
90
91
  config_param :utc_index, :bool, :default => true
91
92
  config_param :type_name, :string, :default => DEFAULT_TYPE_NAME
93
+ config_param :suppress_type_name, :bool, :default => false
92
94
  config_param :index_name, :string, :default => "fluentd"
93
95
  config_param :id_key, :string, :default => nil
94
96
  config_param :write_operation, :string, :default => "index"
@@ -143,6 +145,7 @@ see: https://github.com/elastic/elasticsearch-ruby/pull/514
143
145
  EOC
144
146
  config_param :include_index_in_url, :bool, :default => false
145
147
  config_param :http_backend, :enum, list: [:excon, :typhoeus], :default => :excon
148
+ config_param :http_backend_excon_nonblock, :bool, :default => true
146
149
  config_param :validate_client_version, :bool, :default => false
147
150
  config_param :prefer_oj_serializer, :bool, :default => false
148
151
  config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "es_rejected_execution_exception"]
@@ -158,6 +161,7 @@ EOC
158
161
  config_param :enable_ilm, :bool, :default => false
159
162
  config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
160
163
  config_param :ilm_policy, :hash, :default => {}
164
+ config_param :ilm_policies, :hash, :default => {}
161
165
  config_param :ilm_policy_overwrite, :bool, :default => false
162
166
  config_param :truncate_caches_interval, :time, :default => nil
163
167
 
@@ -218,6 +222,11 @@ EOC
218
222
  log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
219
223
  end
220
224
 
225
+ raise Fluent::ConfigError, "You can't specify ilm_policy and ilm_policies at the same time" unless @ilm_policy.empty? or @ilm_policies.empty?
226
+
227
+ unless @ilm_policy.empty?
228
+ @ilm_policies = { @ilm_policy_id => @ilm_policy }
229
+ end
221
230
  @alias_indexes = []
222
231
  @template_names = []
223
232
  if !dry_run?
@@ -226,14 +235,14 @@ EOC
226
235
  raise Fluent::ConfigError, "deflector_alias is prohibited to use with 'logstash_format at same time." if @logstash_format and @deflector_alias
227
236
  end
228
237
  if @ilm_policy.empty? && @ilm_policy_overwrite
229
- raise Fluent::ConfigError, "ilm_policy_overwrite can work with non empty ilm_policy. Specify non-empty ilm policy into ilm_policy. "
238
+ raise Fluent::ConfigError, "ilm_policy_overwrite requires a non empty ilm_policy."
230
239
  end
231
240
  if @logstash_format || placeholder_substitution_needed_for_template?
232
241
  class << self
233
242
  alias_method :template_installation, :template_installation_actual
234
243
  end
235
244
  else
236
- template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
245
+ template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
237
246
  end
238
247
  verify_ilm_working if @enable_ilm
239
248
  elsif @templates
@@ -296,16 +305,20 @@ EOC
296
305
  else
297
306
  @default_elasticsearch_version
298
307
  end
299
- if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
300
- log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
301
- end
302
- if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
303
- log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
304
- @type_name = '_doc'.freeze
305
- end
306
- if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
307
- log.info "Detected ES 8.x or above: This parameter has no effect."
308
+ if @suppress_type_name && @last_seen_major_version >= 7
308
309
  @type_name = nil
310
+ else
311
+ if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
312
+ log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
313
+ end
314
+ if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
315
+ log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
316
+ @type_name = '_doc'.freeze
317
+ end
318
+ if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
319
+ log.info "Detected ES 8.x or above: This parameter has no effect."
320
+ @type_name = nil
321
+ end
309
322
  end
310
323
 
311
324
  if @validate_client_version && !dry_run?
@@ -327,6 +340,13 @@ EOC
327
340
  end
328
341
  end
329
342
 
343
+ if @ssl_version && @scheme == :https
344
+ if !@http_backend_excon_nonblock
345
+ log.warn "TLS handshake will be stucked with block connection.
346
+ Consider to set `http_backend_excon_nonblock` as true"
347
+ end
348
+ end
349
+
330
350
  # Consider missing the prefix of "$." in nested key specifiers.
331
351
  @id_key = convert_compat_id_key(@id_key) if @id_key
332
352
  @parent_key = convert_compat_id_key(@parent_key) if @parent_key
@@ -335,6 +355,7 @@ EOC
335
355
  @routing_key_name = configure_routing_key_name
336
356
  @meta_config_map = create_meta_config_map
337
357
  @current_config = nil
358
+ @compressable_connection = false
338
359
 
339
360
  @ignore_exception_classes = @ignore_exceptions.map do |exception|
340
361
  unless Object.const_defined?(exception)
@@ -409,7 +430,7 @@ EOC
409
430
  def backend_options
410
431
  case @http_backend
411
432
  when :excon
412
- { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
433
+ { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
413
434
  when :typhoeus
414
435
  require 'typhoeus'
415
436
  { sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
@@ -503,13 +524,15 @@ EOC
503
524
  return Time.at(event_time).to_datetime
504
525
  end
505
526
 
506
- def client(host = nil)
527
+ def client(host = nil, compress_connection = false)
507
528
  # check here to see if we already have a client connection for the given host
508
529
  connection_options = get_connection_options(host)
509
530
 
510
531
  @_es = nil unless is_existing_connection(connection_options[:hosts])
532
+ @_es = nil unless @compressable_connection == compress_connection
511
533
 
512
534
  @_es ||= begin
535
+ @compressable_connection = compress_connection
513
536
  @current_config = connection_options[:hosts].clone
514
537
  adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
515
538
  local_reload_connections = @reload_connections
@@ -517,7 +540,7 @@ EOC
517
540
  local_reload_connections = @reload_after
518
541
  end
519
542
 
520
- gzip_headers = if compression
543
+ gzip_headers = if compress_connection
521
544
  {'Content-Encoding' => 'gzip'}
522
545
  else
523
546
  {}
@@ -543,7 +566,7 @@ EOC
543
566
  },
544
567
  sniffer_class: @sniffer_class,
545
568
  serializer_class: @serializer_class,
546
- compression: compression,
569
+ compression: compress_connection,
547
570
  }), &adapter_conf)
548
571
  Elasticsearch::Client.new transport: transport
549
572
  end
@@ -707,7 +730,12 @@ EOC
707
730
  else
708
731
  pipeline = nil
709
732
  end
710
- return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
733
+ if @ilm_policy_id
734
+ ilm_policy_id = extract_placeholders(@ilm_policy_id, chunk)
735
+ else
736
+ ilm_policy_id = nil
737
+ end
738
+ return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline, ilm_policy_id
711
739
  end
712
740
 
713
741
  def multi_workers_ready?
@@ -733,9 +761,9 @@ EOC
733
761
  begin
734
762
  meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
735
763
  info = if @include_index_in_url
736
- RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze])
764
+ RequestInfo.new(host, meta.delete("_index".freeze), meta.delete("_alias".freeze))
737
765
  else
738
- RequestInfo.new(host, nil, meta["_index".freeze])
766
+ RequestInfo.new(host, nil, meta.delete("_alias".freeze))
739
767
  end
740
768
 
741
769
  if split_request?(bulk_message, info)
@@ -781,7 +809,7 @@ EOC
781
809
  end
782
810
 
783
811
  def process_message(tag, meta, header, time, record, extracted_values)
784
- logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
812
+ logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
785
813
 
786
814
  if @flatten_hashes
787
815
  record = flatten_record(record)
@@ -804,17 +832,19 @@ EOC
804
832
 
805
833
  target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
806
834
  if target_index_parent && target_index_parent[target_index_child_key]
807
- target_index = target_index_parent.delete(target_index_child_key)
835
+ target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
808
836
  elsif @logstash_format
809
837
  dt = dt.new_offset(0) if @utc_index
810
838
  target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
839
+ target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
811
840
  else
812
- target_index = index_name
841
+ target_index_alias = target_index = index_name
813
842
  end
814
843
 
815
844
  # Change target_index to lower-case since Elasticsearch doesn't
816
845
  # allow upper-case characters in index names.
817
846
  target_index = target_index.downcase
847
+ target_index_alias = target_index_alias.downcase
818
848
  if @include_tag_key
819
849
  record[@tag_key] = tag
820
850
  end
@@ -833,7 +863,9 @@ EOC
833
863
  target_type = nil
834
864
  end
835
865
  else
836
- if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
866
+ if @suppress_type_name && @last_seen_major_version >= 7
867
+ target_type = nil
868
+ elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
837
869
  log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
838
870
  target_type = '_doc'.freeze
839
871
  elsif @last_seen_major_version >= 8
@@ -847,6 +879,7 @@ EOC
847
879
  meta.clear
848
880
  meta["_index".freeze] = target_index
849
881
  meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
882
+ meta["_alias".freeze] = target_index_alias
850
883
 
851
884
  if @pipeline
852
885
  meta["pipeline".freeze] = pipeline
@@ -889,16 +922,17 @@ EOC
889
922
  placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
890
923
  placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
891
924
  placeholder?(:deflector_alias, @deflector_alias.to_s) ||
892
- placeholder?(:application_name, @application_name.to_s)
925
+ placeholder?(:application_name, @application_name.to_s) ||
926
+ placeholder?(:ilm_policy_id, @ilm_policy_id.to_s)
893
927
  log.debug("Need substitution: #{need_substitution}")
894
928
  need_substitution
895
929
  end
896
930
 
897
- def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
931
+ def template_installation(deflector_alias, template_name, customize_template, application_name, ilm_policy_id, target_index, host)
898
932
  # for safety.
899
933
  end
900
934
 
901
- def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
935
+ def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
902
936
  if template_name && @template_file
903
937
  if @alias_indexes.include? deflector_alias
904
938
  log.debug("Index alias #{deflector_alias} already exists (cached)")
@@ -907,11 +941,12 @@ EOC
907
941
  else
908
942
  retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
909
943
  if customize_template
910
- template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, host)
944
+ template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
911
945
  else
912
- template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, host)
946
+ template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
913
947
  end
914
- create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, @ilm_policy_overwrite, host)
948
+ ilm_policy = @ilm_policies[ilm_policy_id] || {}
949
+ create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
915
950
  end
916
951
  @alias_indexes << deflector_alias unless deflector_alias.nil?
917
952
  @template_names << template_name unless template_name.nil?
@@ -922,11 +957,11 @@ EOC
922
957
  # send_bulk given a specific bulk request, the original tag,
923
958
  # chunk, and bulk_message_count
924
959
  def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
925
- logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
960
+ logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
926
961
  if deflector_alias
927
- template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
962
+ template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
928
963
  else
929
- template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
964
+ template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
930
965
  end
931
966
 
932
967
  begin
@@ -939,7 +974,7 @@ EOC
939
974
  data
940
975
  end
941
976
 
942
- response = client(info.host).bulk body: prepared_data, index: info.index
977
+ response = client(info.host, compression).bulk body: prepared_data, index: info.index
943
978
  log.on_trace { log.trace "bulk response: #{response}" }
944
979
 
945
980
  if response['errors']
@@ -35,16 +35,18 @@ module Fluent::Plugin
35
35
  end
36
36
 
37
37
 
38
- def client(host = nil)
38
+ def client(host = nil, compress_connection = false)
39
39
  # check here to see if we already have a client connection for the given host
40
40
  connection_options = get_connection_options(host)
41
41
 
42
42
  @_es = nil unless is_existing_connection(connection_options[:hosts])
43
+ @_es = nil unless @compressable_connection == compress_connection
43
44
 
44
45
  @_es ||= begin
46
+ @compressable_connection = compress_connection
45
47
  @current_config = connection_options[:hosts].clone
46
48
  adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
47
- gzip_headers = if compression
49
+ gzip_headers = if compress_connection
48
50
  {'Content-Encoding' => 'gzip'}
49
51
  else
50
52
  {}
@@ -67,7 +69,7 @@ module Fluent::Plugin
67
69
  password: @password,
68
70
  scheme: @scheme
69
71
  },
70
- compression: compression,
72
+ compression: compress_connection,
71
73
  }), &adapter_conf)
72
74
  Elasticsearch::Client.new transport: transport
73
75
  end
@@ -228,7 +230,7 @@ module Fluent::Plugin
228
230
  else
229
231
  data
230
232
  end
231
- response = client(host).bulk body: prepared_data, index: index
233
+ response = client(host, compression).bulk body: prepared_data, index: index
232
234
  if response['errors']
233
235
  log.error "Could not push log to Elasticsearch: #{response}"
234
236
  end
@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
62
62
  d = driver('')
63
63
  ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
64
64
  if @use_tls_minmax_version
65
- assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION,
66
- min_version: OpenSSL::SSL::TLS1_VERSION}, ssl_version_options)
65
+ assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
66
+ min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
67
67
  else
68
68
  assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
69
69
  end
@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
18
18
  Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
19
19
  end
20
20
 
21
+ test "invalid configuration" do
22
+ assert_raise(Fluent::ConfigError) do
23
+ create_driver("use_record_as_seed true")
24
+ end
25
+ end
26
+
21
27
  def sample_record
22
28
  {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
23
29
  end
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
41
47
  assert_equal(Base64.strict_encode64(SecureRandom.uuid),
42
48
  d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
43
49
  end
50
+
51
+ class UseRecordAsSeedTest < self
52
+ data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
53
+ "sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
54
+ "sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
55
+ "sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
56
+ )
57
+ def test_simple(data)
58
+ hash_type, expected = data
59
+ d = create_driver(%[
60
+ use_record_as_seed true
61
+ record_keys age,parent_id,routing_id,custom_key
62
+ hash_type #{hash_type}
63
+ ])
64
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
65
+ d.run(default_tag: 'test') do
66
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
67
+ end
68
+ assert_equal(expected,
69
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
70
+ end
71
+
72
+ data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
73
+ "sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
74
+ "sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
75
+ "sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
76
+ def test_record_with_tag(data)
77
+ hash_type, expected = data
78
+ d = create_driver(%[
79
+ use_record_as_seed true
80
+ record_keys age,parent_id,routing_id,custom_key
81
+ hash_type #{hash_type}
82
+ include_tag_in_seed true
83
+ ])
84
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
85
+ d.run(default_tag: 'test.fluentd') do
86
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
87
+ end
88
+ assert_equal(expected,
89
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
90
+ end
91
+
92
+ data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
93
+ "sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
94
+ "sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
95
+ "sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
96
+ def test_record_with_time(data)
97
+ hash_type, expected = data
98
+ d = create_driver(%[
99
+ use_record_as_seed true
100
+ record_keys age,parent_id,routing_id,custom_key
101
+ hash_type #{hash_type}
102
+ include_time_in_seed true
103
+ ])
104
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
105
+ d.run(default_tag: 'test.fluentd') do
106
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
107
+ end
108
+ assert_equal(expected,
109
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
110
+ end
111
+
112
+ data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
113
+ "sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
114
+ "sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
115
+ "sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
116
+ def test_record_with_tag_and_time
117
+ hash_type, expected = data
118
+ d = create_driver(%[
119
+ use_record_as_seed true
120
+ record_keys age,parent_id,routing_id,custom_key
121
+ hash_type #{hash_type}
122
+ include_tag_in_seed true
123
+ include_time_in_seed true
124
+ ])
125
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
126
+ d.run(default_tag: 'test.fluentd') do
127
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
128
+ end
129
+ assert_equal(expected,
130
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
131
+ end
132
+ end
133
+
134
+ class UseEntireRecordAsSeedTest < self
135
+ data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
136
+ "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
137
+ "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
138
+ "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
139
+ def test_record
140
+ hash_type, expected = data
141
+ d = create_driver(%[
142
+ use_record_as_seed true
143
+ use_entire_record true
144
+ hash_type #{hash_type}
145
+ ])
146
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
147
+ d.run(default_tag: 'test.fluentd') do
148
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
149
+ end
150
+ assert_equal(expected,
151
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
152
+ end
153
+
154
+ data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
155
+ "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
156
+ "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
157
+ "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
158
+ def test_record_with_tag
159
+ hash_type, expected = data
160
+ d = create_driver(%[
161
+ use_record_as_seed true
162
+ use_entire_record true
163
+ hash_type #{hash_type}
164
+ include_tag_in_seed true
165
+ ])
166
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
167
+ d.run(default_tag: 'test.fluentd') do
168
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
169
+ end
170
+ assert_equal(expected,
171
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
172
+ end
173
+
174
+ data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
175
+ "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
176
+ "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
177
+ "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
178
+ def test_record_with_time
179
+ hash_type, expected = data
180
+ d = create_driver(%[
181
+ use_record_as_seed true
182
+ use_entire_record true
183
+ hash_type #{hash_type}
184
+ include_time_in_seed true
185
+ ])
186
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
187
+ d.run(default_tag: 'test.fluentd') do
188
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
189
+ end
190
+ assert_equal(expected,
191
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
192
+ end
193
+
194
+ data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
195
+ "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
196
+ "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
197
+ "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
198
+ def test_record_with_tag_and_time
199
+ hash_type, expected = data
200
+ d = create_driver(%[
201
+ use_record_as_seed true
202
+ use_entire_record true
203
+ hash_type #{hash_type}
204
+ include_tag_in_seed true
205
+ include_time_in_seed true
206
+ ])
207
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
208
+ d.run(default_tag: 'test.fluentd') do
209
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
210
+ end
211
+ assert_equal(expected,
212
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
213
+ end
214
+ end
44
215
  end