fluent-plugin-elasticsearch 4.1.1 → 5.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
  4. data/.github/dependabot.yml +6 -0
  5. data/.github/workflows/issue-auto-closer.yml +2 -2
  6. data/.github/workflows/linux.yml +5 -2
  7. data/.github/workflows/macos.yml +5 -2
  8. data/.github/workflows/windows.yml +5 -2
  9. data/Gemfile +1 -2
  10. data/History.md +146 -0
  11. data/README.ElasticsearchGenID.md +4 -4
  12. data/README.ElasticsearchInput.md +1 -1
  13. data/README.Troubleshooting.md +692 -0
  14. data/README.md +260 -550
  15. data/fluent-plugin-elasticsearch.gemspec +4 -1
  16. data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
  17. data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
  18. data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
  19. data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
  20. data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
  21. data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
  22. data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
  23. data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
  24. data/lib/fluent/plugin/oj_serializer.rb +2 -1
  25. data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
  26. data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
  27. data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
  28. data/test/helper.rb +0 -4
  29. data/test/plugin/mock_chunk.dat +0 -0
  30. data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
  31. data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
  32. data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
  33. data/test/plugin/test_elasticsearch_tls.rb +8 -2
  34. data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
  35. data/test/plugin/test_in_elasticsearch.rb +51 -21
  36. data/test/plugin/test_index_alias_template.json +11 -0
  37. data/test/plugin/test_index_template.json +25 -0
  38. data/test/plugin/test_out_elasticsearch.rb +2118 -704
  39. data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
  40. data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
  41. metadata +62 -10
  42. data/.coveralls.yml +0 -2
  43. data/.travis.yml +0 -44
  44. data/appveyor.yml +0 -20
  45. data/gemfiles/Gemfile.without.ilm +0 -10
@@ -2,23 +2,24 @@
2
2
  require 'date'
3
3
  require 'excon'
4
4
  require 'elasticsearch'
5
- begin
6
- require 'elasticsearch/xpack'
7
- rescue LoadError
8
- end
5
+ require 'set'
9
6
  require 'json'
10
7
  require 'uri'
8
+ require 'base64'
11
9
  begin
12
10
  require 'strptime'
13
11
  rescue LoadError
14
12
  end
13
+ require 'resolv'
15
14
 
16
15
  require 'fluent/plugin/output'
17
16
  require 'fluent/event'
18
17
  require 'fluent/error'
19
18
  require 'fluent/time'
19
+ require 'fluent/unique_id'
20
20
  require 'fluent/log-ext'
21
21
  require 'zlib'
22
+ require_relative 'elasticsearch_compat'
22
23
  require_relative 'elasticsearch_constants'
23
24
  require_relative 'elasticsearch_error'
24
25
  require_relative 'elasticsearch_error_handler'
@@ -31,6 +32,8 @@ begin
31
32
  rescue LoadError
32
33
  end
33
34
 
35
+ require 'faraday/excon'
36
+
34
37
  module Fluent::Plugin
35
38
  class ElasticsearchOutput < Output
36
39
  class RecoverableRequestFailure < StandardError; end
@@ -57,6 +60,7 @@ module Fluent::Plugin
57
60
  attr_reader :template_names
58
61
  attr_reader :ssl_version_options
59
62
  attr_reader :compressable_connection
63
+ attr_reader :api_key_header
60
64
 
61
65
  helpers :event_emitter, :compat_parameters, :record_accessor, :timer
62
66
 
@@ -67,13 +71,15 @@ module Fluent::Plugin
67
71
  DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
68
72
  DEFAULT_TYPE_NAME = "fluentd".freeze
69
73
  DEFAULT_RELOAD_AFTER = -1
70
- TARGET_BULK_BYTES = 20 * 1024 * 1024
74
+ DEFAULT_TARGET_BULK_BYTES = -1
71
75
  DEFAULT_POLICY_ID = "logstash-policy"
72
76
 
73
77
  config_param :host, :string, :default => 'localhost'
74
78
  config_param :port, :integer, :default => 9200
75
79
  config_param :user, :string, :default => nil
76
80
  config_param :password, :string, :default => nil, :secret => true
81
+ config_param :cloud_id, :string, :default => nil
82
+ config_param :cloud_auth, :string, :default => nil
77
83
  config_param :path, :string, :default => nil
78
84
  config_param :scheme, :enum, :list => [:https, :http], :default => :http
79
85
  config_param :hosts, :string, :default => nil
@@ -155,10 +161,11 @@ EOC
155
161
  config_param :default_elasticsearch_version, :integer, :default => DEFAULT_ELASTICSEARCH_VERSION
156
162
  config_param :log_es_400_reason, :bool, :default => false
157
163
  config_param :custom_headers, :hash, :default => {}
164
+ config_param :api_key, :string, :default => nil, :secret => true
158
165
  config_param :suppress_doc_wrap, :bool, :default => false
159
166
  config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
160
167
  config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
161
- config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
168
+ config_param :bulk_message_request_threshold, :size, :default => DEFAULT_TARGET_BULK_BYTES
162
169
  config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
163
170
  config_param :enable_ilm, :bool, :default => false
164
171
  config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
@@ -166,6 +173,14 @@ EOC
166
173
  config_param :ilm_policies, :hash, :default => {}
167
174
  config_param :ilm_policy_overwrite, :bool, :default => false
168
175
  config_param :truncate_caches_interval, :time, :default => nil
176
+ config_param :use_legacy_template, :bool, :default => true
177
+ config_param :catch_transport_exception_on_retry, :bool, :default => true
178
+ config_param :target_index_affinity, :bool, :default => false
179
+
180
+ config_section :metadata, param_name: :metainfo, multi: false do
181
+ config_param :include_chunk_id, :bool, :default => false
182
+ config_param :chunk_id_key, :string, :default => "chunk_id".freeze
183
+ end
169
184
 
170
185
  config_section :buffer do
171
186
  config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -212,6 +227,8 @@ EOC
212
227
  @remove_keys_on_update = @remove_keys_on_update.split ','
213
228
  end
214
229
 
230
+ @api_key_header = setup_api_key
231
+
215
232
  raise Fluent::ConfigError, "'max_retry_putting_template' must be greater than or equal to zero." if @max_retry_putting_template < 0
216
233
  raise Fluent::ConfigError, "'max_retry_get_es_version' must be greater than or equal to zero." if @max_retry_get_es_version < 0
217
234
 
@@ -247,8 +264,11 @@ EOC
247
264
  template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
248
265
  end
249
266
  verify_ilm_working if @enable_ilm
250
- elsif @templates
251
- retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
267
+ end
268
+ if @templates
269
+ retry_operate(@max_retry_putting_template,
270
+ @fail_on_putting_template_retry_exceed,
271
+ @catch_transport_exception_on_retry) do
252
272
  templates_hash_install(@templates, @template_overwrite)
253
273
  end
254
274
  end
@@ -278,8 +298,14 @@ EOC
278
298
  @dump_proc = Yajl.method(:dump)
279
299
  end
280
300
 
301
+ raise Fluent::ConfigError, "`cloud_auth` must be present if `cloud_id` is present" if @cloud_id && @cloud_auth.nil?
281
302
  raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
282
303
 
304
+ if @cloud_auth
305
+ @user = @cloud_auth.split(':', -1)[0]
306
+ @password = @cloud_auth.split(':', -1)[1]
307
+ end
308
+
283
309
  if @user && m = @user.match(/%{(?<user>.*)}/)
284
310
  @user = URI.encode_www_form_component(m["user"])
285
311
  end
@@ -325,7 +351,7 @@ EOC
325
351
  @type_name = '_doc'.freeze
326
352
  end
327
353
  if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
328
- log.info "Detected ES 8.x or above: This parameter has no effect."
354
+ log.debug "Detected ES 8.x or above: This parameter has no effect."
329
355
  @type_name = nil
330
356
  end
331
357
  end
@@ -386,17 +412,23 @@ EOC
386
412
  end
387
413
  end
388
414
 
389
- if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
415
+ if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
390
416
  if compression
391
417
  raise Fluent::ConfigError, <<-EOC
392
418
  Cannot use compression with elasticsearch-transport plugin version < 7.2.0
393
- Your elasticsearch-transport plugin version version is #{Elasticsearch::Transport::VERSION}.
419
+ Your elasticsearch-transport plugin version version is #{TRANSPORT_CLASS::VERSION}.
394
420
  Please consider to upgrade ES client.
395
421
  EOC
396
422
  end
397
423
  end
398
424
  end
399
425
 
426
+ def setup_api_key
427
+ return {} unless @api_key
428
+
429
+ { "Authorization" => "ApiKey " + Base64.strict_encode64(@api_key) }
430
+ end
431
+
400
432
  def dry_run?
401
433
  if Fluent::Engine.respond_to?(:dry_run_mode)
402
434
  Fluent::Engine.dry_run_mode
@@ -441,7 +473,7 @@ EOC
441
473
  when :excon
442
474
  { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
443
475
  when :typhoeus
444
- require 'typhoeus'
476
+ require 'faraday/typhoeus'
445
477
  { sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
446
478
  end
447
479
  rescue LoadError => ex
@@ -451,7 +483,9 @@ EOC
451
483
 
452
484
  def handle_last_seen_es_major_version
453
485
  if @verify_es_version_at_startup && !dry_run?
454
- retry_operate(@max_retry_get_es_version, @fail_on_detecting_es_version_retry_exceed) do
486
+ retry_operate(@max_retry_get_es_version,
487
+ @fail_on_detecting_es_version_retry_exceed,
488
+ @catch_transport_exception_on_retry) do
455
489
  detect_es_major_version
456
490
  end
457
491
  else
@@ -460,8 +494,20 @@ EOC
460
494
  end
461
495
 
462
496
  def detect_es_major_version
463
- @_es_info ||= client.info
464
- @_es_info["version"]["number"].to_i
497
+ begin
498
+ @_es_info ||= client.info
499
+ rescue ::Elasticsearch::UnsupportedProductError => e
500
+ raise Fluent::ConfigError, "Using Elasticsearch client #{client_library_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
501
+ end
502
+ begin
503
+ unless version = @_es_info.dig("version", "number")
504
+ version = @default_elasticsearch_version
505
+ end
506
+ rescue NoMethodError => e
507
+ log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
508
+ version = @default_elasticsearch_version
509
+ end
510
+ version.to_i
465
511
  end
466
512
 
467
513
  def client_library_version
@@ -533,7 +579,17 @@ EOC
533
579
  return Time.at(event_time).to_datetime
534
580
  end
535
581
 
582
+ def cloud_client
583
+ Elasticsearch::Client.new(
584
+ cloud_id: @cloud_id,
585
+ user: @user,
586
+ password: @password
587
+ )
588
+ end
589
+
536
590
  def client(host = nil, compress_connection = false)
591
+ return cloud_client if @cloud_id
592
+
537
593
  # check here to see if we already have a client connection for the given host
538
594
  connection_options = get_connection_options(host)
539
595
 
@@ -554,10 +610,13 @@ EOC
554
610
  else
555
611
  {}
556
612
  end
557
- headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers).merge(gzip_headers)
613
+ headers = { 'Content-Type' => @content_type.to_s }
614
+ .merge(@custom_headers)
615
+ .merge(@api_key_header)
616
+ .merge(gzip_headers)
558
617
  ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
559
618
 
560
- transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
619
+ transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
561
620
  options: {
562
621
  reload_connections: local_reload_connections,
563
622
  reload_on_failure: @reload_on_failure,
@@ -594,6 +653,14 @@ EOC
594
653
  end
595
654
  end
596
655
 
656
+ def is_ipv6_host(host_str)
657
+ begin
658
+ IPAddr.new(host_str).ipv6?
659
+ rescue IPAddr::InvalidAddressError
660
+ return false
661
+ end
662
+ end
663
+
597
664
  def get_connection_options(con_host=nil)
598
665
 
599
666
  hosts = if con_host || @hosts
@@ -605,6 +672,21 @@ EOC
605
672
  port: (host_str.split(':')[1] || @port).to_i,
606
673
  scheme: @scheme.to_s
607
674
  }
675
+ # Support ipv6 for host/host placeholders
676
+ elsif is_ipv6_host(host_str)
677
+ if Resolv::IPv6::Regex.match(host_str)
678
+ {
679
+ host: "[#{host_str}]",
680
+ port: @port.to_i,
681
+ scheme: @scheme.to_s
682
+ }
683
+ else
684
+ {
685
+ host: host_str,
686
+ port: @port.to_i,
687
+ scheme: @scheme.to_s
688
+ }
689
+ end
608
690
  else
609
691
  # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
610
692
  uri = URI(get_escaped_userinfo(host_str))
@@ -615,7 +697,11 @@ EOC
615
697
  end
616
698
  end.compact
617
699
  else
618
- [{host: @host, port: @port, scheme: @scheme.to_s}]
700
+ if Resolv::IPv6::Regex.match(@host)
701
+ [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
702
+ else
703
+ [{host: @host, port: @port, scheme: @scheme.to_s}]
704
+ end
619
705
  end.each do |host|
620
706
  host.merge!(user: @user, password: @password) if !host[:user] && @user
621
707
  host.merge!(path: @path) if !host[:path] && @path
@@ -752,13 +838,24 @@ EOC
752
838
  true
753
839
  end
754
840
 
841
+ def inject_chunk_id_to_record_if_needed(record, chunk_id)
842
+ if @metainfo&.include_chunk_id
843
+ record[@metainfo.chunk_id_key] = chunk_id
844
+ record
845
+ else
846
+ record
847
+ end
848
+ end
849
+
755
850
  def write(chunk)
756
851
  bulk_message_count = Hash.new { |h,k| h[k] = 0 }
757
852
  bulk_message = Hash.new { |h,k| h[k] = '' }
758
853
  header = {}
759
854
  meta = {}
855
+ unpackedMsgArr = {}
760
856
 
761
857
  tag = chunk.metadata.tag
858
+ chunk_id = dump_unique_id_hex(chunk.unique_id)
762
859
  extracted_values = expand_placeholders(chunk)
763
860
  host = if @hosts
764
861
  extract_placeholders(@hosts, chunk)
@@ -766,19 +863,27 @@ EOC
766
863
  extract_placeholders(@host, chunk)
767
864
  end
768
865
 
866
+ affinity_target_indices = get_affinity_target_indices(chunk)
769
867
  chunk.msgpack_each do |time, record|
770
868
  next unless record.is_a? Hash
869
+
870
+ record = inject_chunk_id_to_record_if_needed(record, chunk_id)
871
+
771
872
  begin
772
- meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
873
+ meta, header, record = process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
773
874
  info = if @include_index_in_url
774
875
  RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
775
876
  else
776
877
  RequestInfo.new(host, nil, meta["_index".freeze], meta.delete("_alias".freeze))
777
878
  end
778
879
 
880
+ unpackedMsgArr[info] = [] if unpackedMsgArr[info].nil?
881
+ unpackedMsgArr[info] << {:time => time, :record => record}
882
+
779
883
  if split_request?(bulk_message, info)
780
884
  bulk_message.each do |info, msgs|
781
- send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
885
+ send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
886
+ unpackedMsgArr[info].clear
782
887
  msgs.clear
783
888
  # Clear bulk_message_count for this info.
784
889
  bulk_message_count[info] = 0;
@@ -801,11 +906,49 @@ EOC
801
906
  end
802
907
 
803
908
  bulk_message.each do |info, msgs|
804
- send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
909
+ send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
910
+
911
+ unpackedMsgArr[info].clear
805
912
  msgs.clear
806
913
  end
807
914
  end
808
915
 
916
+ def target_index_affinity_enabled?()
917
+ @target_index_affinity && @logstash_format && @id_key && (@write_operation == UPDATE_OP || @write_operation == UPSERT_OP)
918
+ end
919
+
920
+ def get_affinity_target_indices(chunk)
921
+ indices = Hash.new
922
+ if target_index_affinity_enabled?()
923
+ id_key_accessor = record_accessor_create(@id_key)
924
+ ids = Set.new
925
+ chunk.msgpack_each do |time, record|
926
+ next unless record.is_a? Hash
927
+ begin
928
+ ids << id_key_accessor.call(record)
929
+ end
930
+ end
931
+ log.debug("Find affinity target_indices by quering on ES (write_operation #{@write_operation}) for ids: #{ids.to_a}")
932
+ options = {
933
+ :index => "#{logstash_prefix}#{@logstash_prefix_separator}*",
934
+ }
935
+ query = {
936
+ 'query' => { 'ids' => { 'values' => ids.to_a } },
937
+ '_source' => false,
938
+ 'sort' => [
939
+ {"_index" => {"order" => "desc"}}
940
+ ]
941
+ }
942
+ result = client.search(options.merge(:body => Yajl.dump(query)))
943
+ # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
944
+ result['hits']['hits'].each do |hit|
945
+ indices[hit["_id"]] = hit["_index"]
946
+ log.debug("target_index for id: #{hit["_id"]} from es: #{hit["_index"]}")
947
+ end
948
+ end
949
+ indices
950
+ end
951
+
809
952
  def split_request?(bulk_message, info)
810
953
  # For safety.
811
954
  end
@@ -818,7 +961,7 @@ EOC
818
961
  false
819
962
  end
820
963
 
821
- def process_message(tag, meta, header, time, record, extracted_values)
964
+ def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
822
965
  logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
823
966
 
824
967
  if @flatten_hashes
@@ -859,6 +1002,15 @@ EOC
859
1002
  record[@tag_key] = tag
860
1003
  end
861
1004
 
1005
+ # If affinity target indices map has value for this particular id, use it as target_index
1006
+ if !affinity_target_indices.empty?
1007
+ id_accessor = record_accessor_create(@id_key)
1008
+ id_value = id_accessor.call(record)
1009
+ if affinity_target_indices.key?(id_value)
1010
+ target_index = affinity_target_indices[id_value]
1011
+ end
1012
+ end
1013
+
862
1014
  target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
863
1015
  if target_type_parent && target_type_parent[target_type_child_key]
864
1016
  target_type = target_type_parent.delete(target_type_child_key)
@@ -868,18 +1020,18 @@ EOC
868
1020
  elsif @last_seen_major_version == 7
869
1021
  log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
870
1022
  target_type = '_doc'.freeze
871
- elsif @last_seen_major_version >=8
872
- log.warn "Detected ES 8.x or above: document type will not be used."
1023
+ elsif @last_seen_major_version >= 8
1024
+ log.debug "Detected ES 8.x or above: document type will not be used."
873
1025
  target_type = nil
874
1026
  end
875
1027
  else
876
- if @suppress_type_name && @last_seen_major_version >= 7
1028
+ if @suppress_type_name && @last_seen_major_version == 7
877
1029
  target_type = nil
878
1030
  elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
879
1031
  log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
880
1032
  target_type = '_doc'.freeze
881
1033
  elsif @last_seen_major_version >= 8
882
- log.warn "Detected ES 8.x or above: document type will not be used."
1034
+ log.debug "Detected ES 8.x or above: document type will not be used."
883
1035
  target_type = nil
884
1036
  else
885
1037
  target_type = type_name
@@ -944,29 +1096,33 @@ EOC
944
1096
 
945
1097
  def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
946
1098
  if template_name && @template_file
947
- if !@logstash_format && @alias_indexes.include?(deflector_alias)
948
- log.debug("Index alias #{deflector_alias} already exists (cached)")
949
- elsif !@logstash_format && @template_names.include?(template_name)
950
- log.debug("Template name #{template_name} already exists (cached)")
1099
+ if !@logstash_format && (deflector_alias.nil? || (@alias_indexes.include? deflector_alias)) && (@template_names.include? template_name)
1100
+ if deflector_alias
1101
+ log.debug("Index alias #{deflector_alias} and template #{template_name} already exist (cached)")
1102
+ else
1103
+ log.debug("Template #{template_name} already exists (cached)")
1104
+ end
951
1105
  else
952
- retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
1106
+ retry_operate(@max_retry_putting_template,
1107
+ @fail_on_putting_template_retry_exceed,
1108
+ @catch_transport_exception_on_retry) do
953
1109
  if customize_template
954
- template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
1110
+ template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
955
1111
  else
956
- template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
1112
+ template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
957
1113
  end
958
1114
  ilm_policy = @ilm_policies[ilm_policy_id] || {}
959
1115
  create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
960
1116
  end
961
1117
  @alias_indexes << deflector_alias unless deflector_alias.nil?
962
- @template_names << template_name unless template_name.nil?
1118
+ @template_names << template_name
963
1119
  end
964
1120
  end
965
1121
  end
966
1122
 
967
1123
  # send_bulk given a specific bulk request, the original tag,
968
1124
  # chunk, and bulk_message_count
969
- def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
1125
+ def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info, unpacked_msg_arr)
970
1126
  _logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
971
1127
  if deflector_alias
972
1128
  template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
@@ -989,7 +1145,7 @@ EOC
989
1145
 
990
1146
  if response['errors']
991
1147
  error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
992
- error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
1148
+ error.handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
993
1149
  end
994
1150
  rescue RetryStreamError => e
995
1151
  log.trace "router.emit_stream for retry stream doing..."