fluent-plugin-input-opensearch 1.1.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +1 -0
  3. data/.editorconfig +9 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.md +29 -0
  5. data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
  6. data/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +9 -0
  7. data/.github/workflows/coverage.yaml +22 -0
  8. data/.github/workflows/issue-auto-closer.yml +12 -0
  9. data/.github/workflows/linux.yml +26 -0
  10. data/.github/workflows/macos.yml +26 -0
  11. data/.github/workflows/windows.yml +26 -0
  12. data/.gitignore +18 -0
  13. data/CONTRIBUTING.md +24 -0
  14. data/Gemfile +10 -0
  15. data/History.md +67 -0
  16. data/LICENSE.txt +201 -0
  17. data/README.OpenSearchGenID.md +116 -0
  18. data/README.OpenSearchInput.md +314 -0
  19. data/README.Troubleshooting.md +482 -0
  20. data/README.md +1622 -0
  21. data/Rakefile +37 -0
  22. data/fluent-plugin-opensearch.gemspec +39 -0
  23. data/gemfiles/Gemfile.elasticsearch.v6 +12 -0
  24. data/lib/fluent/log-ext.rb +64 -0
  25. data/lib/fluent/plugin/filter_opensearch_genid.rb +103 -0
  26. data/lib/fluent/plugin/in_opensearch.rb +410 -0
  27. data/lib/fluent/plugin/oj_serializer.rb +48 -0
  28. data/lib/fluent/plugin/opensearch_constants.rb +39 -0
  29. data/lib/fluent/plugin/opensearch_error.rb +31 -0
  30. data/lib/fluent/plugin/opensearch_error_handler.rb +182 -0
  31. data/lib/fluent/plugin/opensearch_fallback_selector.rb +36 -0
  32. data/lib/fluent/plugin/opensearch_index_template.rb +155 -0
  33. data/lib/fluent/plugin/opensearch_simple_sniffer.rb +36 -0
  34. data/lib/fluent/plugin/opensearch_tls.rb +96 -0
  35. data/lib/fluent/plugin/out_opensearch.rb +1158 -0
  36. data/lib/fluent/plugin/out_opensearch_data_stream.rb +229 -0
  37. data/test/helper.rb +60 -0
  38. data/test/plugin/datastream_template.json +4 -0
  39. data/test/plugin/test_alias_template.json +9 -0
  40. data/test/plugin/test_filter_opensearch_genid.rb +241 -0
  41. data/test/plugin/test_in_opensearch.rb +500 -0
  42. data/test/plugin/test_index_alias_template.json +11 -0
  43. data/test/plugin/test_index_template.json +25 -0
  44. data/test/plugin/test_oj_serializer.rb +45 -0
  45. data/test/plugin/test_opensearch_error_handler.rb +770 -0
  46. data/test/plugin/test_opensearch_fallback_selector.rb +100 -0
  47. data/test/plugin/test_opensearch_tls.rb +171 -0
  48. data/test/plugin/test_out_opensearch.rb +3980 -0
  49. data/test/plugin/test_out_opensearch_data_stream.rb +746 -0
  50. data/test/plugin/test_template.json +23 -0
  51. data/test/test_log-ext.rb +61 -0
  52. metadata +291 -0
@@ -0,0 +1,1158 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'date'
28
+ require 'excon'
29
+ require 'opensearch'
30
+ require 'set'
31
+ require 'json'
32
+ require 'uri'
33
+ require 'base64'
34
+ begin
35
+ require 'strptime'
36
+ rescue LoadError
37
+ end
38
+ require 'resolv'
39
+
40
+ require 'fluent/plugin/output'
41
+ require 'fluent/event'
42
+ require 'fluent/error'
43
+ require 'fluent/time'
44
+ require 'fluent/unique_id'
45
+ require 'fluent/log-ext'
46
+ require 'zlib'
47
+ require_relative 'opensearch_constants'
48
+ require_relative 'opensearch_error'
49
+ require_relative 'opensearch_error_handler'
50
+ require_relative 'opensearch_index_template'
51
+ require_relative 'opensearch_tls'
52
+ require_relative 'opensearch_fallback_selector'
53
+ begin
54
+ require_relative 'oj_serializer'
55
+ rescue LoadError
56
+ end
57
+ require 'aws-sdk-core'
58
+ require 'faraday_middleware/aws_sigv4'
59
+ require 'faraday/excon'
60
+
61
+ module Fluent::Plugin
62
+ class OpenSearchOutput < Output
63
+ class RecoverableRequestFailure < StandardError; end
64
+ class UnrecoverableRequestFailure < Fluent::UnrecoverableError; end
65
+ class RetryStreamEmitFailure < StandardError; end
66
+
67
+ # MissingIdFieldError is raised for records that do not
68
+ # include the field for the unique record identifier
69
+ class MissingIdFieldError < StandardError; end
70
+
71
+ # RetryStreamError privides a stream to be
72
+ # put back in the pipeline for cases where a bulk request
73
+ # failed (e.g some records succeed while others failed)
74
+ class RetryStreamError < StandardError
75
+ attr_reader :retry_stream
76
+ def initialize(retry_stream)
77
+ @retry_stream = retry_stream
78
+ end
79
+ end
80
+
81
+ RequestInfo = Struct.new(:host, :index, :target_index, :alias)
82
+
83
+ attr_reader :template_names
84
+ attr_reader :ssl_version_options
85
+ attr_reader :compressable_connection
86
+
87
+ helpers :event_emitter, :compat_parameters, :record_accessor, :timer
88
+
89
+ Fluent::Plugin.register_output('opensearch', self)
90
+
91
+ DEFAULT_BUFFER_TYPE = "memory"
92
+ DEFAULT_OPENSEARCH_VERSION = 1
93
+ DEFAULT_TYPE_NAME = "_doc".freeze
94
+ DEFAULT_RELOAD_AFTER = -1
95
+ DEFAULT_TARGET_BULK_BYTES = -1
96
+ DEFAULT_POLICY_ID = "logstash-policy"
97
+
98
+ config_param :host, :string, :default => 'localhost'
99
+ config_param :port, :integer, :default => 9200
100
+ config_param :user, :string, :default => nil
101
+ config_param :password, :string, :default => nil, :secret => true
102
+ config_param :path, :string, :default => nil
103
+ config_param :scheme, :enum, :list => [:https, :http], :default => :http
104
+ config_param :hosts, :string, :default => nil
105
+ config_param :target_index_key, :string, :default => nil
106
+ config_param :time_key_format, :string, :default => nil
107
+ config_param :time_precision, :integer, :default => 9
108
+ config_param :include_timestamp, :bool, :default => false
109
+ config_param :logstash_format, :bool, :default => false
110
+ config_param :logstash_prefix, :string, :default => "logstash"
111
+ config_param :logstash_prefix_separator, :string, :default => '-'
112
+ config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
113
+ config_param :utc_index, :bool, :default => true
114
+ config_param :suppress_type_name, :bool, :default => false
115
+ config_param :index_name, :string, :default => "fluentd"
116
+ config_param :id_key, :string, :default => nil
117
+ config_param :write_operation, :string, :default => "index"
118
+ config_param :parent_key, :string, :default => nil
119
+ config_param :routing_key, :string, :default => nil
120
+ config_param :request_timeout, :time, :default => 5
121
+ config_param :reload_connections, :bool, :default => true
122
+ config_param :reload_on_failure, :bool, :default => false
123
+ config_param :retry_tag, :string, :default=>nil
124
+ config_param :resurrect_after, :time, :default => 60
125
+ config_param :time_key, :string, :default => nil
126
+ config_param :time_key_exclude_timestamp, :bool, :default => false
127
+ config_param :ssl_verify , :bool, :default => true
128
+ config_param :client_key, :string, :default => nil
129
+ config_param :client_cert, :string, :default => nil
130
+ config_param :client_key_pass, :string, :default => nil, :secret => true
131
+ config_param :ca_file, :string, :default => nil
132
+ config_param :remove_keys, :string, :default => nil
133
+ config_param :remove_keys_on_update, :string, :default => ""
134
+ config_param :remove_keys_on_update_key, :string, :default => nil
135
+ config_param :flatten_hashes, :bool, :default => false
136
+ config_param :flatten_hashes_separator, :string, :default => "_"
137
+ config_param :template_name, :string, :default => nil
138
+ config_param :template_file, :string, :default => nil
139
+ config_param :template_overwrite, :bool, :default => false
140
+ config_param :customize_template, :hash, :default => nil
141
+ config_param :index_date_pattern, :string, :default => "now/d"
142
+ config_param :index_separator, :string, :default => "-"
143
+ config_param :application_name, :string, :default => "default"
144
+ config_param :templates, :hash, :default => nil
145
+ config_param :max_retry_putting_template, :integer, :default => 10
146
+ config_param :fail_on_putting_template_retry_exceed, :bool, :default => true
147
+ config_param :fail_on_detecting_os_version_retry_exceed, :bool, :default => true
148
+ config_param :max_retry_get_os_version, :integer, :default => 15
149
+ config_param :include_tag_key, :bool, :default => false
150
+ config_param :tag_key, :string, :default => 'tag'
151
+ config_param :time_parse_error_tag, :string, :default => 'opensearch_plugin.output.time.error'
152
+ config_param :reconnect_on_error, :bool, :default => false
153
+ config_param :pipeline, :string, :default => nil
154
+ config_param :with_transporter_log, :bool, :default => false
155
+ config_param :emit_error_for_missing_id, :bool, :default => false
156
+ config_param :sniffer_class_name, :string, :default => nil
157
+ config_param :selector_class_name, :string, :default => nil
158
+ config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
159
+ config_param :include_index_in_url, :bool, :default => false
160
+ config_param :http_backend, :enum, list: [:excon, :typhoeus], :default => :excon
161
+ config_param :http_backend_excon_nonblock, :bool, :default => true
162
+ config_param :validate_client_version, :bool, :default => false
163
+ config_param :prefer_oj_serializer, :bool, :default => false
164
+ config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "rejected_execution_exception"]
165
+ config_param :unrecoverable_record_types, :array, :default => ["json_parse_exception"]
166
+ config_param :emit_error_label_event, :bool, :default => true
167
+ config_param :verify_os_version_at_startup, :bool, :default => true
168
+ config_param :default_opensearch_version, :integer, :default => DEFAULT_OPENSEARCH_VERSION
169
+ config_param :log_os_400_reason, :bool, :default => false
170
+ config_param :custom_headers, :hash, :default => {}
171
+ config_param :suppress_doc_wrap, :bool, :default => false
172
+ config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
173
+ config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
174
+ config_param :bulk_message_request_threshold, :size, :default => DEFAULT_TARGET_BULK_BYTES
175
+ config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
176
+ config_param :truncate_caches_interval, :time, :default => nil
177
+ config_param :use_legacy_template, :bool, :default => true
178
+ config_param :catch_transport_exception_on_retry, :bool, :default => true
179
+ config_param :target_index_affinity, :bool, :default => false
180
+
181
+ config_section :metadata, param_name: :metainfo, multi: false do
182
+ config_param :include_chunk_id, :bool, :default => false
183
+ config_param :chunk_id_key, :string, :default => "chunk_id".freeze
184
+ end
185
+
186
+ config_section :endpoint, multi: false do
187
+ config_param :region, :string
188
+ config_param :url do |c|
189
+ c.chomp("/")
190
+ end
191
+ config_param :access_key_id, :string, :default => ""
192
+ config_param :secret_access_key, :string, :default => "", secret: true
193
+ config_param :assume_role_arn, :string, :default => nil
194
+ config_param :ecs_container_credentials_relative_uri, :string, :default => nil #Set with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable value
195
+ config_param :assume_role_session_name, :string, :default => "fluentd"
196
+ config_param :assume_role_web_identity_token_file, :string, :default => nil
197
+ config_param :sts_credentials_region, :string, :default => nil
198
+ config_param :refresh_credentials_interval, :time, :default => "5h"
199
+ config_param :aws_service_name, :enum, list: [:es, :aoss], :default => :es
200
+ end
201
+
202
+ config_section :buffer do
203
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
204
+ config_set_default :chunk_keys, ['tag']
205
+ config_set_default :timekey_use_utc, true
206
+ end
207
+
208
+ include Fluent::OpenSearchIndexTemplate
209
+ include Fluent::Plugin::OpenSearchConstants
210
+ include Fluent::Plugin::OpenSearchTLS
211
+
212
+ def initialize
213
+ super
214
+ end
215
+
216
+ ######################################################################################################
217
+ # This creating AWS credentials code part is heavily based on fluent-plugin-aws-elasticsearch-service:
218
+ # https://github.com/atomita/fluent-plugin-aws-elasticsearch-service/blob/master/lib/fluent/plugin/out_aws-elasticsearch-service.rb#L73-L134
219
+ ######################################################################################################
220
+ def aws_credentials(conf)
221
+ credentials = nil
222
+ unless conf[:access_key_id].empty? || conf[:secret_access_key].empty?
223
+ credentials = Aws::Credentials.new(conf[:access_key_id], conf[:secret_access_key])
224
+ else
225
+ if conf[:assume_role_arn].nil?
226
+ aws_container_credentials_relative_uri = conf[:ecs_container_credentials_relative_uri] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
227
+ if aws_container_credentials_relative_uri.nil?
228
+ credentials = Aws::SharedCredentials.new({retries: 2}).credentials rescue nil
229
+ credentials ||= Aws::InstanceProfileCredentials.new.credentials rescue nil
230
+ credentials ||= Aws::ECSCredentials.new.credentials
231
+ else
232
+ credentials = Aws::ECSCredentials.new({
233
+ credential_path: aws_container_credentials_relative_uri
234
+ }).credentials
235
+ end
236
+ else
237
+ if conf[:assume_role_web_identity_token_file].nil?
238
+ credentials = Aws::AssumeRoleCredentials.new({
239
+ role_arn: conf[:assume_role_arn],
240
+ role_session_name: conf[:assume_role_session_name],
241
+ region: sts_creds_region(conf)
242
+ }).credentials
243
+ else
244
+ credentials = Aws::AssumeRoleWebIdentityCredentials.new({
245
+ role_arn: conf[:assume_role_arn],
246
+ web_identity_token_file: conf[:assume_role_web_identity_token_file],
247
+ region: sts_creds_region(conf)
248
+ }).credentials
249
+ end
250
+ end
251
+ end
252
+ raise "No valid AWS credentials found." unless credentials.set?
253
+
254
+ credentials
255
+ end
256
+
257
+ def sts_creds_region(conf)
258
+ conf[:sts_credentials_region] || conf[:region]
259
+ end
260
+ ###############################
261
+ # AWS credential part is ended.
262
+ ###############################
263
+
264
+ def configure(conf)
265
+ compat_parameters_convert(conf, :buffer)
266
+
267
+ super
268
+
269
+ if @endpoint
270
+ # here overrides default value of reload_connections to false because
271
+ # AWS Elasticsearch Service doesn't return addresses of nodes and Elasticsearch client
272
+ # fails to reload connections properly. This ends up "temporarily failed to flush the buffer"
273
+ # error repeating forever. See this discussion for details:
274
+ # https://discuss.elastic.co/t/elasitcsearch-ruby-raises-cannot-get-new-connection-from-pool-error/36252
275
+ @reload_connections = false
276
+ end
277
+
278
+ if placeholder_substitution_needed_for_template?
279
+ # nop.
280
+ elsif not @buffer_config.chunk_keys.include? "tag" and
281
+ not @buffer_config.chunk_keys.include? "_index"
282
+ raise Fluent::ConfigError, "'tag' or '_index' in chunk_keys is required."
283
+ end
284
+ @time_parser = create_time_parser
285
+ @backend_options = backend_options
286
+ @ssl_version_options = set_tls_minmax_version_config(@ssl_version, @ssl_max_version, @ssl_min_version)
287
+
288
+ if @remove_keys
289
+ @remove_keys = @remove_keys.split(/\s*,\s*/)
290
+ end
291
+
292
+ if @target_index_key && @target_index_key.is_a?(String)
293
+ @target_index_key = @target_index_key.split '.'
294
+ end
295
+
296
+ if @remove_keys_on_update && @remove_keys_on_update.is_a?(String)
297
+ @remove_keys_on_update = @remove_keys_on_update.split ','
298
+ end
299
+
300
+ raise Fluent::ConfigError, "'max_retry_putting_template' must be greater than or equal to zero." if @max_retry_putting_template < 0
301
+ raise Fluent::ConfigError, "'max_retry_get_os_version' must be greater than or equal to zero." if @max_retry_get_os_version < 0
302
+
303
+ # Dump log when using host placeholders and template features at same time.
304
+ valid_host_placeholder = placeholder?(:host_placeholder, @host)
305
+ if valid_host_placeholder && (@template_name && @template_file || @templates)
306
+ if @verify_os_version_at_startup
307
+ raise Fluent::ConfigError, "host placeholder, template installation, and verify OpenSearch version at startup are exclusive feature at same time. Please specify verify_os_version_at_startup as `false` when host placeholder and template installation are enabled."
308
+ end
309
+ log.info "host placeholder and template installation makes your OpenSearch cluster a bit slow down(beta)."
310
+ end
311
+
312
+ @template_names = []
313
+ if !dry_run?
314
+ if @template_name && @template_file
315
+ if @logstash_format || placeholder_substitution_needed_for_template?
316
+ class << self
317
+ alias_method :template_installation, :template_installation_actual
318
+ end
319
+ else
320
+ template_installation_actual(@template_name, @customize_template, @application_name, @index_name)
321
+ end
322
+ end
323
+ if @templates
324
+ retry_operate(@max_retry_putting_template,
325
+ @fail_on_putting_template_retry_exceed,
326
+ @catch_transport_exception_on_retry) do
327
+ templates_hash_install(@templates, @template_overwrite)
328
+ end
329
+ end
330
+ end
331
+
332
+ @truncate_mutex = Mutex.new
333
+ if @truncate_caches_interval
334
+ timer_execute(:out_opensearch_truncate_caches, @truncate_caches_interval) do
335
+ log.info('Clean up the indices and template names cache')
336
+
337
+ @truncate_mutex.synchronize {
338
+ @template_names.clear
339
+ }
340
+ end
341
+ end
342
+ # If AWS credentials is set, consider to expire credentials information forcibly before expired.
343
+ @credential_mutex = Mutex.new
344
+ if @endpoint
345
+ @_aws_credentials = aws_credentials(@endpoint)
346
+
347
+ if @endpoint.refresh_credentials_interval
348
+ timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
349
+ log.debug('Recreate the AWS credentials')
350
+
351
+ @credential_mutex.synchronize do
352
+ @_os = nil
353
+ @_aws_credentials = aws_credentials(@endpoint)
354
+ end
355
+ end
356
+ end
357
+ end
358
+
359
+ @serializer_class = nil
360
+ begin
361
+ require 'oj'
362
+ @dump_proc = Oj.method(:dump)
363
+ if @prefer_oj_serializer
364
+ @serializer_class = Fluent::Plugin::Serializer::Oj
365
+ OpenSearch::API.settings[:serializer] = Fluent::Plugin::Serializer::Oj
366
+ end
367
+ rescue LoadError
368
+ @dump_proc = Yajl.method(:dump)
369
+ end
370
+
371
+ raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
372
+
373
+ if @user && m = @user.match(/%{(?<user>.*)}/)
374
+ @user = URI.encode_www_form_component(m["user"])
375
+ end
376
+ if @password && m = @password.match(/%{(?<password>.*)}/)
377
+ @password = URI.encode_www_form_component(m["password"])
378
+ end
379
+
380
+ @transport_logger = nil
381
+ if @with_transporter_log
382
+ @transport_logger = log
383
+ log_level = conf['@log_level'] || conf['log_level']
384
+ log.warn "Consider to specify log_level with @log_level." unless log_level
385
+ end
386
+ # Specify @sniffer_class before calling #client.
387
+ # #detect_os_major_version uses #client.
388
+ @sniffer_class = nil
389
+ begin
390
+ @sniffer_class = Object.const_get(@sniffer_class_name) if @sniffer_class_name
391
+ rescue Exception => ex
392
+ raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
393
+ end
394
+
395
+ @selector_class = nil
396
+ begin
397
+ @selector_class = Object.const_get(@selector_class_name) if @selector_class_name
398
+ rescue Exception => ex
399
+ raise Fluent::ConfigError, "Could not load selector class #{@selector_class_name}: #{ex}"
400
+ end
401
+
402
+ @last_seen_major_version = if major_version = handle_last_seen_os_major_version
403
+ major_version
404
+ else
405
+ @default_opensearch_version
406
+ end
407
+
408
+ if @validate_client_version && !dry_run?
409
+ if @last_seen_major_version != client_library_version.to_i
410
+ raise Fluent::ConfigError, <<-EOC
411
+ Detected OpenSearch #{@last_seen_major_version} but you use OpenSearch client #{client_library_version}.
412
+ Please consider to use #{@last_seen_major_version}.x series OpenSearch client.
413
+ EOC
414
+ end
415
+ end
416
+
417
+ if @last_seen_major_version >= 1
418
+ case @ssl_version
419
+ when :SSLv23, :TLSv1, :TLSv1_1
420
+ if @scheme == :https
421
+ log.warn "Detected OpenSearch 1.x or above and enabled insecure security:
422
+ You might have to specify `ssl_version TLSv1_2` in configuration."
423
+ end
424
+ end
425
+ end
426
+
427
+ if @ssl_version && @scheme == :https
428
+ if !@http_backend_excon_nonblock
429
+ log.warn "TLS handshake will be stucked with block connection.
430
+ Consider to set `http_backend_excon_nonblock` as true"
431
+ end
432
+ end
433
+
434
+ # Consider missing the prefix of "$." in nested key specifiers.
435
+ @id_key = convert_compat_id_key(@id_key) if @id_key
436
+ @parent_key = convert_compat_id_key(@parent_key) if @parent_key
437
+ @routing_key = convert_compat_id_key(@routing_key) if @routing_key
438
+
439
+ @routing_key_name = configure_routing_key_name
440
+ @meta_config_map = create_meta_config_map
441
+ @current_config = nil
442
+ @compressable_connection = false
443
+
444
+ @ignore_exception_classes = @ignore_exceptions.map do |exception|
445
+ unless Object.const_defined?(exception)
446
+ log.warn "Cannot find class #{exception}. Will ignore it."
447
+
448
+ nil
449
+ else
450
+ Object.const_get(exception)
451
+ end
452
+ end.compact
453
+
454
+ if @bulk_message_request_threshold < 0
455
+ class << self
456
+ alias_method :split_request?, :split_request_size_uncheck?
457
+ end
458
+ else
459
+ class << self
460
+ alias_method :split_request?, :split_request_size_check?
461
+ end
462
+ end
463
+ end
464
+
465
+ def dry_run?
466
+ if Fluent::Engine.respond_to?(:dry_run_mode)
467
+ Fluent::Engine.dry_run_mode
468
+ elsif Fluent::Engine.respond_to?(:supervisor_mode)
469
+ Fluent::Engine.supervisor_mode
470
+ end
471
+ end
472
+
473
+ def placeholder?(name, param)
474
+ placeholder_validities = []
475
+ placeholder_validators(name, param).each do |v|
476
+ begin
477
+ v.validate!
478
+ placeholder_validities << true
479
+ rescue Fluent::ConfigError => e
480
+ log.debug("'#{name} #{param}' is tested built-in placeholder(s) but there is no valid placeholder(s). error: #{e}")
481
+ placeholder_validities << false
482
+ end
483
+ end
484
+ placeholder_validities.include?(true)
485
+ end
486
+
487
+ def emit_error_label_event?
488
+ !!@emit_error_label_event
489
+ end
490
+
491
+ def compression
492
+ !(@compression_level == :no_compression)
493
+ end
494
+
495
+ def compression_strategy
496
+ case @compression_level
497
+ when :default_compression
498
+ Zlib::DEFAULT_COMPRESSION
499
+ when :best_compression
500
+ Zlib::BEST_COMPRESSION
501
+ when :best_speed
502
+ Zlib::BEST_SPEED
503
+ else
504
+ Zlib::NO_COMPRESSION
505
+ end
506
+ end
507
+
508
+ def backend_options
509
+ case @http_backend
510
+ when :excon
511
+ { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
512
+ when :typhoeus
513
+ require 'faraday/typhoeus'
514
+ { sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
515
+ end
516
+ rescue LoadError => ex
517
+ log.error_backtrace(ex.backtrace)
518
+ raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
519
+ end
520
+
521
+ def handle_last_seen_os_major_version
522
+ if @verify_os_version_at_startup && !dry_run?
523
+ retry_operate(@max_retry_get_os_version,
524
+ @fail_on_detecting_os_version_retry_exceed,
525
+ @catch_transport_exception_on_retry) do
526
+ detect_os_major_version
527
+ end
528
+ else
529
+ nil
530
+ end
531
+ end
532
+
533
+ def detect_os_major_version
534
+ @_os_info ||= client.info
535
+ begin
536
+ unless version = @_os_info.dig("version", "number")
537
+ version = @default_opensearch_version
538
+ end
539
+ rescue NoMethodError => e
540
+ log.warn "#{@_os_info} can not dig version information. Assuming OpenSearch #{@default_opensearch_version}", error: e
541
+ version = @default_opensearch_version
542
+ end
543
+ version.to_i
544
+ end
545
+
546
+ def client_library_version
547
+ OpenSearch::VERSION
548
+ end
549
+
550
+ def configure_routing_key_name
551
+ 'routing'.freeze
552
+ end
553
+
554
+ def convert_compat_id_key(key)
555
+ if key.include?('.') && !key.start_with?('$[')
556
+ key = "$.#{key}" unless key.start_with?('$.')
557
+ end
558
+ key
559
+ end
560
+
561
+ def create_meta_config_map
562
+ result = []
563
+ result << [record_accessor_create(@id_key), '_id'] if @id_key
564
+ result << [record_accessor_create(@parent_key), '_parent'] if @parent_key
565
+ result << [record_accessor_create(@routing_key), @routing_key_name] if @routing_key
566
+ result
567
+ end
568
+
569
+ # once fluent v0.14 is released we might be able to use
570
+ # Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives
571
+ # [sec,nsec] where as we want something we can call `strftime` on...
572
+ def create_time_parser
573
+ if @time_key_format
574
+ begin
575
+ # Strptime doesn't support all formats, but for those it does it's
576
+ # blazingly fast.
577
+ strptime = Strptime.new(@time_key_format)
578
+ Proc.new { |value|
579
+ value = convert_numeric_time_into_string(value, @time_key_format) if value.is_a?(Numeric)
580
+ strptime.exec(value).to_datetime
581
+ }
582
+ rescue
583
+ # Can happen if Strptime doesn't recognize the format; or
584
+ # if strptime couldn't be required (because it's not installed -- it's
585
+ # ruby 2 only)
586
+ Proc.new { |value|
587
+ value = convert_numeric_time_into_string(value, @time_key_format) if value.is_a?(Numeric)
588
+ DateTime.strptime(value, @time_key_format)
589
+ }
590
+ end
591
+ else
592
+ Proc.new { |value|
593
+ value = convert_numeric_time_into_string(value) if value.is_a?(Numeric)
594
+ DateTime.parse(value)
595
+ }
596
+ end
597
+ end
598
+
599
+ def convert_numeric_time_into_string(numeric_time, time_key_format = "%Y-%m-%d %H:%M:%S.%N%z")
600
+ numeric_time_parser = Fluent::NumericTimeParser.new(:float)
601
+ Time.at(numeric_time_parser.parse(numeric_time).to_r).strftime(time_key_format)
602
+ end
603
+
604
+ def parse_time(value, event_time, tag)
605
+ @time_parser.call(value)
606
+ rescue => e
607
+ if emit_error_label_event?
608
+ router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
609
+ end
610
+ return Time.at(event_time).to_datetime
611
+ end
612
+
613
+ def client(host = nil, compress_connection = false)
614
+ # check here to see if we already have a client connection for the given host
615
+ connection_options = get_connection_options(host)
616
+
617
+ @_os = nil unless is_existing_connection(connection_options[:hosts])
618
+ @_os = nil unless @compressable_connection == compress_connection
619
+
620
+ @_os ||= begin
621
+ @compressable_connection = compress_connection
622
+ @current_config = connection_options[:hosts].clone
623
+ adapter_conf = if @endpoint
624
+ lambda do |f|
625
+ f.request(
626
+ :aws_sigv4,
627
+ service: @endpoint.aws_service_name.to_s,
628
+ region: @endpoint.region,
629
+ credentials: @_aws_credentials,
630
+ )
631
+
632
+ f.adapter @http_backend, @backend_options
633
+ end
634
+ else
635
+ lambda {|f| f.adapter @http_backend, @backend_options }
636
+ end
637
+
638
+ local_reload_connections = @reload_connections
639
+ if local_reload_connections && @reload_after > DEFAULT_RELOAD_AFTER
640
+ local_reload_connections = @reload_after
641
+ end
642
+
643
+ gzip_headers = if compress_connection
644
+ {'Content-Encoding' => 'gzip'}
645
+ else
646
+ {}
647
+ end
648
+ headers = {}.merge(@custom_headers)
649
+ .merge(gzip_headers)
650
+ ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
651
+
652
+ transport = OpenSearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
653
+ options: {
654
+ reload_connections: local_reload_connections,
655
+ reload_on_failure: @reload_on_failure,
656
+ resurrect_after: @resurrect_after,
657
+ logger: @transport_logger,
658
+ transport_options: {
659
+ headers: headers,
660
+ request: { timeout: @request_timeout },
661
+ ssl: ssl_options,
662
+ },
663
+ http: {
664
+ user: @user,
665
+ password: @password,
666
+ scheme: @scheme
667
+ },
668
+ sniffer_class: @sniffer_class,
669
+ serializer_class: @serializer_class,
670
+ selector_class: @selector_class,
671
+ compression: compress_connection,
672
+ }), &adapter_conf)
673
+ OpenSearch::Client.new transport: transport
674
+ end
675
+ end
676
+
677
+ def get_escaped_userinfo(host_str)
678
+ if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
679
+ m["scheme"] +
680
+ URI.encode_www_form_component(m["user"]) +
681
+ ':' +
682
+ URI.encode_www_form_component(m["password"]) +
683
+ m["path"]
684
+ else
685
+ host_str
686
+ end
687
+ end
688
+
689
+ def get_connection_options(con_host=nil)
690
+
691
+ hosts = if @endpoint # For AWS OpenSearch Service
692
+ uri = URI(@endpoint.url)
693
+ host = %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
694
+ hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
695
+ hash
696
+ end
697
+ [host]
698
+ elsif con_host || @hosts
699
+ (con_host || @hosts).split(',').map do |host_str|
700
+ # Support legacy hosts format host:port,host:port,host:port...
701
+ if host_str.match(%r{^[^:]+(\:\d+)?$})
702
+ {
703
+ host: host_str.split(':')[0],
704
+ port: (host_str.split(':')[1] || @port).to_i,
705
+ scheme: @scheme.to_s
706
+ }
707
+ else
708
+ # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
709
+ uri = URI(get_escaped_userinfo(host_str))
710
+ %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
711
+ hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
712
+ hash
713
+ end
714
+ end
715
+ end.compact
716
+ else
717
+ if Resolv::IPv6::Regex.match(@host)
718
+ [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
719
+ else
720
+ [{host: @host, port: @port, scheme: @scheme.to_s}]
721
+ end
722
+ end.each do |host|
723
+ host.merge!(user: @user, password: @password) if !host[:user] && @user
724
+ host.merge!(path: @path) if !host[:path] && @path
725
+ end
726
+
727
+ {
728
+ hosts: hosts
729
+ }
730
+ end
731
+
732
+ def connection_options_description(con_host=nil)
733
+ get_connection_options(con_host)[:hosts].map do |host_info|
734
+ attributes = host_info.dup
735
+ attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
736
+ attributes.inspect
737
+ end.join(', ')
738
+ end
739
+
740
+ # append_record_to_messages adds a record to the bulk message
741
+ # payload to be submitted to OpenSearch. Records that do
742
+ # not include '_id' field are skipped when 'write_operation'
743
+ # is configured for 'create' or 'update'
744
+ #
745
+ # returns 'true' if record was appended to the bulk message
746
+ # and 'false' otherwise
747
+ def append_record_to_messages(op, meta, header, record, msgs)
748
+ case op
749
+ when UPDATE_OP, UPSERT_OP
750
+ if meta.has_key?(ID_FIELD)
751
+ header[UPDATE_OP] = meta
752
+ msgs << @dump_proc.call(header) << BODY_DELIMITER
753
+ msgs << @dump_proc.call(update_body(record, op)) << BODY_DELIMITER
754
+ return true
755
+ end
756
+ when CREATE_OP
757
+ if meta.has_key?(ID_FIELD)
758
+ header[CREATE_OP] = meta
759
+ msgs << @dump_proc.call(header) << BODY_DELIMITER
760
+ msgs << @dump_proc.call(record) << BODY_DELIMITER
761
+ return true
762
+ end
763
+ when INDEX_OP
764
+ header[INDEX_OP] = meta
765
+ msgs << @dump_proc.call(header) << BODY_DELIMITER
766
+ msgs << @dump_proc.call(record) << BODY_DELIMITER
767
+ return true
768
+ end
769
+ return false
770
+ end
771
+
772
+ def update_body(record, op)
773
+ update = remove_keys(record)
774
+ if @suppress_doc_wrap
775
+ return update
776
+ end
777
+ body = {"doc".freeze => update}
778
+ if op == UPSERT_OP
779
+ if update == record
780
+ body["doc_as_upsert".freeze] = true
781
+ else
782
+ body[UPSERT_OP] = record
783
+ end
784
+ end
785
+ body
786
+ end
787
+
788
+ def remove_keys(record)
789
+ keys = record[@remove_keys_on_update_key] || @remove_keys_on_update || []
790
+ record.delete(@remove_keys_on_update_key)
791
+ return record unless keys.any?
792
+ record = record.dup
793
+ keys.each { |key| record.delete(key) }
794
+ record
795
+ end
796
+
797
+ def flatten_record(record, prefix=[])
798
+ ret = {}
799
+ if record.is_a? Hash
800
+ record.each { |key, value|
801
+ ret.merge! flatten_record(value, prefix + [key.to_s])
802
+ }
803
+ elsif record.is_a? Array
804
+ # Don't mess with arrays, leave them unprocessed
805
+ ret.merge!({prefix.join(@flatten_hashes_separator) => record})
806
+ else
807
+ return {prefix.join(@flatten_hashes_separator) => record}
808
+ end
809
+ ret
810
+ end
811
+
812
+ def expand_placeholders(chunk)
813
+ logstash_prefix = extract_placeholders(@logstash_prefix, chunk)
814
+ logstash_dateformat = extract_placeholders(@logstash_dateformat, chunk)
815
+ index_name = extract_placeholders(@index_name, chunk)
816
+ if @template_name
817
+ template_name = extract_placeholders(@template_name, chunk)
818
+ else
819
+ template_name = nil
820
+ end
821
+ if @customize_template
822
+ customize_template = @customize_template.each_with_object({}) { |(key, value), hash| hash[key] = extract_placeholders(value, chunk) }
823
+ else
824
+ customize_template = nil
825
+ end
826
+ if @application_name
827
+ application_name = extract_placeholders(@application_name, chunk)
828
+ else
829
+ application_name = nil
830
+ end
831
+ if @pipeline
832
+ pipeline = extract_placeholders(@pipeline, chunk)
833
+ else
834
+ pipeline = nil
835
+ end
836
+ return logstash_prefix, logstash_dateformat, index_name, template_name, customize_template, application_name, pipeline
837
+ end
838
+
839
+ def multi_workers_ready?
840
+ true
841
+ end
842
+
843
+ def inject_chunk_id_to_record_if_needed(record, chunk_id)
844
+ if @metainfo&.include_chunk_id
845
+ record[@metainfo.chunk_id_key] = chunk_id
846
+ record
847
+ else
848
+ record
849
+ end
850
+ end
851
+
852
+ def write(chunk)
853
+ bulk_message_count = Hash.new { |h,k| h[k] = 0 }
854
+ bulk_message = Hash.new { |h,k| h[k] = '' }
855
+ header = {}
856
+ meta = {}
857
+
858
+ tag = chunk.metadata.tag
859
+ chunk_id = dump_unique_id_hex(chunk.unique_id)
860
+ extracted_values = expand_placeholders(chunk)
861
+ host = if @hosts
862
+ extract_placeholders(@hosts, chunk)
863
+ else
864
+ extract_placeholders(@host, chunk)
865
+ end
866
+
867
+ affinity_target_indices = get_affinity_target_indices(chunk)
868
+ chunk.msgpack_each do |time, record|
869
+ next unless record.is_a? Hash
870
+
871
+ record = inject_chunk_id_to_record_if_needed(record, chunk_id)
872
+
873
+ begin
874
+ meta, header, record = process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
875
+ info = if @include_index_in_url
876
+ RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
877
+ else
878
+ RequestInfo.new(host, nil, meta["_index".freeze], meta.delete("_alias".freeze))
879
+ end
880
+
881
+ if split_request?(bulk_message, info)
882
+ bulk_message.each do |info, msgs|
883
+ send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
884
+ msgs.clear
885
+ # Clear bulk_message_count for this info.
886
+ bulk_message_count[info] = 0;
887
+ next
888
+ end
889
+ end
890
+
891
+ if append_record_to_messages(@write_operation, meta, header, record, bulk_message[info])
892
+ bulk_message_count[info] += 1;
893
+ else
894
+ if @emit_error_for_missing_id
895
+ raise MissingIdFieldError, "Missing '_id' field. Write operation is #{@write_operation}"
896
+ else
897
+ log.on_debug { log.debug("Dropping record because its missing an '_id' field and write_operation is #{@write_operation}: #{record}") }
898
+ end
899
+ end
900
+ rescue => e
901
+ if emit_error_label_event?
902
+ router.emit_error_event(tag, time, record, e)
903
+ end
904
+ end
905
+ end
906
+
907
+ bulk_message.each do |info, msgs|
908
+ send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
909
+ msgs.clear
910
+ end
911
+ end
912
+
913
+ def target_index_affinity_enabled?()
914
+ @target_index_affinity && @logstash_format && @id_key && (@write_operation == UPDATE_OP || @write_operation == UPSERT_OP)
915
+ end
916
+
917
+ def get_affinity_target_indices(chunk)
918
+ indices = Hash.new
919
+ if target_index_affinity_enabled?()
920
+ id_key_accessor = record_accessor_create(@id_key)
921
+ ids = Set.new
922
+ chunk.msgpack_each do |time, record|
923
+ next unless record.is_a? Hash
924
+ begin
925
+ ids << id_key_accessor.call(record)
926
+ end
927
+ end
928
+ log.debug("Find affinity target_indices by quering on OpenSearch (write_operation #{@write_operation}) for ids: #{ids.to_a}")
929
+ options = {
930
+ :index => "#{logstash_prefix}#{@logstash_prefix_separator}*",
931
+ }
932
+ query = {
933
+ 'query' => { 'ids' => { 'values' => ids.to_a } },
934
+ '_source' => false,
935
+ 'sort' => [
936
+ {"_index" => {"order" => "desc"}}
937
+ ]
938
+ }
939
+ result = client.search(options.merge(:body => Yajl.dump(query)))
940
+ # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
941
+ result['hits']['hits'].each do |hit|
942
+ indices[hit["_id"]] = hit["_index"]
943
+ log.debug("target_index for id: #{hit["_id"]} from es: #{hit["_index"]}")
944
+ end
945
+ end
946
+ indices
947
+ end
948
+
949
+ def split_request?(bulk_message, info)
950
+ # For safety.
951
+ end
952
+
953
+ def split_request_size_check?(bulk_message, info)
954
+ bulk_message[info].size > @bulk_message_request_threshold
955
+ end
956
+
957
+ def split_request_size_uncheck?(bulk_message, info)
958
+ false
959
+ end
960
+
961
+ def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
962
+ logstash_prefix, logstash_dateformat, index_name, _template_name, _customize_template, application_name, pipeline = extracted_values
963
+
964
+ if @flatten_hashes
965
+ record = flatten_record(record)
966
+ end
967
+
968
+ dt = nil
969
+ if @logstash_format || @include_timestamp
970
+ if record.has_key?(TIMESTAMP_FIELD)
971
+ rts = record[TIMESTAMP_FIELD]
972
+ dt = parse_time(rts, time, tag)
973
+ elsif record.has_key?(@time_key)
974
+ rts = record[@time_key]
975
+ dt = parse_time(rts, time, tag)
976
+ record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision) unless @time_key_exclude_timestamp
977
+ else
978
+ dt = Time.at(time).to_datetime
979
+ record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
980
+ end
981
+ end
982
+
983
+ target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
984
+ if target_index_parent && target_index_parent[target_index_child_key]
985
+ target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
986
+ elsif @logstash_format
987
+ dt = dt.new_offset(0) if @utc_index
988
+ target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
989
+ target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
990
+ else
991
+ target_index_alias = target_index = index_name
992
+ end
993
+
994
+ # Change target_index to lower-case since OpenSearch doesn't
995
+ # allow upper-case characters in index names.
996
+ target_index = target_index.downcase
997
+ target_index_alias = target_index_alias.downcase
998
+ if @include_tag_key
999
+ record[@tag_key] = tag
1000
+ end
1001
+
1002
+ # If affinity target indices map has value for this particular id, use it as target_index
1003
+ if !affinity_target_indices.empty?
1004
+ id_accessor = record_accessor_create(@id_key)
1005
+ id_value = id_accessor.call(record)
1006
+ if affinity_target_indices.key?(id_value)
1007
+ target_index = affinity_target_indices[id_value]
1008
+ end
1009
+ end
1010
+
1011
+ if @suppress_type_name || @last_seen_major_version >= 2
1012
+ target_type = nil
1013
+ else
1014
+ # OpenSearch only supports "_doc".
1015
+ target_type = DEFAULT_TYPE_NAME
1016
+ end
1017
+
1018
+ meta.clear
1019
+ meta["_index".freeze] = target_index
1020
+ meta["_type".freeze] = target_type unless target_type.nil?
1021
+ meta["_alias".freeze] = target_index_alias
1022
+
1023
+ if @pipeline
1024
+ meta["pipeline".freeze] = pipeline
1025
+ end
1026
+
1027
+ @meta_config_map.each do |record_accessor, meta_key|
1028
+ if raw_value = record_accessor.call(record)
1029
+ meta[meta_key] = raw_value
1030
+ end
1031
+ end
1032
+
1033
+ if @remove_keys
1034
+ @remove_keys.each { |key| record.delete(key) }
1035
+ end
1036
+
1037
+ return [meta, header, record]
1038
+ end
1039
+
1040
+ # returns [parent, child_key] of child described by path array in record's tree
1041
+ # returns [nil, child_key] if path doesnt exist in record
1042
+ def get_parent_of(record, path)
1043
+ parent_object = path[0..-2].reduce(record) { |a, e| a.is_a?(Hash) ? a[e] : nil }
1044
+ [parent_object, path[-1]]
1045
+ end
1046
+
1047
+ # gzip compress data
1048
+ def gzip(string)
1049
+ wio = StringIO.new("w")
1050
+ w_gz = Zlib::GzipWriter.new(wio, strategy = compression_strategy)
1051
+ w_gz.write(string)
1052
+ w_gz.close
1053
+ wio.string
1054
+ end
1055
+
1056
+ def placeholder_substitution_needed_for_template?
1057
+ need_substitution = placeholder?(:host, @host.to_s) ||
1058
+ placeholder?(:index_name, @index_name.to_s) ||
1059
+ placeholder?(:template_name, @template_name.to_s) ||
1060
+ @customize_template&.values&.any? { |value| placeholder?(:customize_template, value.to_s) } ||
1061
+ placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
1062
+ placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
1063
+ placeholder?(:application_name, @application_name.to_s) ||
1064
+ log.debug("Need substitution: #{need_substitution}")
1065
+ need_substitution
1066
+ end
1067
+
1068
+ def template_installation(template_name, customize_template, application_name, target_index, host)
1069
+ # for safety.
1070
+ end
1071
+
1072
+ def template_installation_actual(template_name, customize_template, application_name, target_index, host=nil)
1073
+ if template_name && @template_file
1074
+ if !@logstash_format && @template_names.include?(template_name)
1075
+ log.debug("Template #{template_name} already exists (cached)")
1076
+ else
1077
+ retry_operate(@max_retry_putting_template,
1078
+ @fail_on_putting_template_retry_exceed,
1079
+ @catch_transport_exception_on_retry) do
1080
+ if customize_template
1081
+ template_custom_install(template_name, @template_file, @template_overwrite, customize_template, host, target_index, @index_separator)
1082
+ else
1083
+ template_install(template_name, @template_file, @template_overwrite, host, target_index, @index_separator)
1084
+ end
1085
+ end
1086
+ @template_names << template_name
1087
+ end
1088
+ end
1089
+ end
1090
+
1091
+ # send_bulk given a specific bulk request, the original tag,
1092
+ # chunk, and bulk_message_count
1093
+ def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
1094
+ _logstash_prefix, _logstash_dateformat, index_name, template_name, customize_template, application_name, _pipeline = extracted_values
1095
+ template_installation(template_name, customize_template, application_name, index_name, info.host)
1096
+
1097
+ begin
1098
+
1099
+ log.on_trace { log.trace "bulk request: #{data}" }
1100
+
1101
+ prepared_data = if compression
1102
+ gzip(data)
1103
+ else
1104
+ data
1105
+ end
1106
+
1107
+ response = client(info.host, compression).bulk body: prepared_data, index: info.index
1108
+ log.on_trace { log.trace "bulk response: #{response}" }
1109
+
1110
+ if response['errors']
1111
+ error = Fluent::Plugin::OpenSearchErrorHandler.new(self)
1112
+ error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
1113
+ end
1114
+ rescue RetryStreamError => e
1115
+ log.trace "router.emit_stream for retry stream doing..."
1116
+ emit_tag = @retry_tag ? @retry_tag : tag
1117
+ # check capacity of buffer space
1118
+ if retry_stream_retryable?
1119
+ router.emit_stream(emit_tag, e.retry_stream)
1120
+ else
1121
+ raise RetryStreamEmitFailure, "buffer is full."
1122
+ end
1123
+ log.trace "router.emit_stream for retry stream done."
1124
+ rescue => e
1125
+ ignore = @ignore_exception_classes.any? { |clazz| e.class <= clazz }
1126
+
1127
+ log.warn "Exception ignored in tag #{tag}: #{e.class.name} #{e.message}" if ignore
1128
+
1129
+ @_os = nil if @reconnect_on_error
1130
+ @_os_info = nil if @reconnect_on_error
1131
+
1132
+ raise UnrecoverableRequestFailure if ignore && @exception_backup
1133
+
1134
+ # FIXME: identify unrecoverable errors and raise UnrecoverableRequestFailure instead
1135
+ raise RecoverableRequestFailure, "could not push logs to OpenSearch cluster (#{connection_options_description(info.host)}): #{e.message}" unless ignore
1136
+ end
1137
+ end
1138
+
1139
+ def retry_stream_retryable?
1140
+ @buffer.storable?
1141
+ end
1142
+
1143
+ def is_existing_connection(host)
1144
+ # check if the host provided match the current connection
1145
+ return false if @_os.nil?
1146
+ return false if @current_config.nil?
1147
+ return false if host.length != @current_config.length
1148
+
1149
+ for i in 0...host.length
1150
+ if !host[i][:host].eql? @current_config[i][:host] || host[i][:port] != @current_config[i][:port]
1151
+ return false
1152
+ end
1153
+ end
1154
+
1155
+ return true
1156
+ end
1157
+ end
1158
+ end