fluent-plugin-input-opensearch 1.1.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +1 -0
  3. data/.editorconfig +9 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.md +29 -0
  5. data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
  6. data/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +9 -0
  7. data/.github/workflows/coverage.yaml +22 -0
  8. data/.github/workflows/issue-auto-closer.yml +12 -0
  9. data/.github/workflows/linux.yml +26 -0
  10. data/.github/workflows/macos.yml +26 -0
  11. data/.github/workflows/windows.yml +26 -0
  12. data/.gitignore +18 -0
  13. data/CONTRIBUTING.md +24 -0
  14. data/Gemfile +10 -0
  15. data/History.md +67 -0
  16. data/LICENSE.txt +201 -0
  17. data/README.OpenSearchGenID.md +116 -0
  18. data/README.OpenSearchInput.md +314 -0
  19. data/README.Troubleshooting.md +482 -0
  20. data/README.md +1622 -0
  21. data/Rakefile +37 -0
  22. data/fluent-plugin-opensearch.gemspec +39 -0
  23. data/gemfiles/Gemfile.elasticsearch.v6 +12 -0
  24. data/lib/fluent/log-ext.rb +64 -0
  25. data/lib/fluent/plugin/filter_opensearch_genid.rb +103 -0
  26. data/lib/fluent/plugin/in_opensearch.rb +410 -0
  27. data/lib/fluent/plugin/oj_serializer.rb +48 -0
  28. data/lib/fluent/plugin/opensearch_constants.rb +39 -0
  29. data/lib/fluent/plugin/opensearch_error.rb +31 -0
  30. data/lib/fluent/plugin/opensearch_error_handler.rb +182 -0
  31. data/lib/fluent/plugin/opensearch_fallback_selector.rb +36 -0
  32. data/lib/fluent/plugin/opensearch_index_template.rb +155 -0
  33. data/lib/fluent/plugin/opensearch_simple_sniffer.rb +36 -0
  34. data/lib/fluent/plugin/opensearch_tls.rb +96 -0
  35. data/lib/fluent/plugin/out_opensearch.rb +1158 -0
  36. data/lib/fluent/plugin/out_opensearch_data_stream.rb +229 -0
  37. data/test/helper.rb +60 -0
  38. data/test/plugin/datastream_template.json +4 -0
  39. data/test/plugin/test_alias_template.json +9 -0
  40. data/test/plugin/test_filter_opensearch_genid.rb +241 -0
  41. data/test/plugin/test_in_opensearch.rb +500 -0
  42. data/test/plugin/test_index_alias_template.json +11 -0
  43. data/test/plugin/test_index_template.json +25 -0
  44. data/test/plugin/test_oj_serializer.rb +45 -0
  45. data/test/plugin/test_opensearch_error_handler.rb +770 -0
  46. data/test/plugin/test_opensearch_fallback_selector.rb +100 -0
  47. data/test/plugin/test_opensearch_tls.rb +171 -0
  48. data/test/plugin/test_out_opensearch.rb +3980 -0
  49. data/test/plugin/test_out_opensearch_data_stream.rb +746 -0
  50. data/test/plugin/test_template.json +23 -0
  51. data/test/test_log-ext.rb +61 -0
  52. metadata +291 -0
@@ -0,0 +1,48 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'oj'
28
+
29
+ module Fluent::Plugin
30
+ module Serializer
31
+
32
+ class Oj
33
+ include OpenSearch::Transport::Transport::Serializer::Base
34
+
35
+ # De-serialize a Hash from JSON string
36
+ #
37
+ def load(string, options={})
38
+ ::Oj.load(string, options)
39
+ end
40
+
41
+ # Serialize a Hash to JSON string
42
+ #
43
+ def dump(object, options={})
44
+ ::Oj.dump(object, options)
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,39 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ module Fluent
28
+ module Plugin
29
+ module OpenSearchConstants
30
+ BODY_DELIMITER = "\n".freeze
31
+ UPDATE_OP = "update".freeze
32
+ UPSERT_OP = "upsert".freeze
33
+ CREATE_OP = "create".freeze
34
+ INDEX_OP = "index".freeze
35
+ ID_FIELD = "_id".freeze
36
+ TIMESTAMP_FIELD = "@timestamp".freeze
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,31 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'fluent/error'
28
+
29
+ class Fluent::Plugin::OpenSearchError
30
+ class RetryableOperationExhaustedFailure < Fluent::UnrecoverableError; end
31
+ end
@@ -0,0 +1,182 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'fluent/event'
28
+ require 'fluent/error'
29
+ require_relative 'opensearch_constants'
30
+
31
+ class Fluent::Plugin::OpenSearchErrorHandler
32
+ include Fluent::Plugin::OpenSearchConstants
33
+
34
+ attr_accessor :bulk_message_count
35
+ class OpenSearchVersionMismatch < Fluent::UnrecoverableError; end
36
+ class OpenSearchSubmitMismatch < Fluent::UnrecoverableError; end
37
+ class OpenSearchRequestAbortError < Fluent::UnrecoverableError; end
38
+ class OpenSearchError < StandardError; end
39
+
40
+ def initialize(plugin)
41
+ @plugin = plugin
42
+ end
43
+
44
+ def unrecoverable_error_types
45
+ @plugin.unrecoverable_error_types
46
+ end
47
+
48
+ def unrecoverable_error?(type)
49
+ unrecoverable_error_types.include?(type)
50
+ end
51
+
52
+ def unrecoverable_record_types
53
+ @plugin.unrecoverable_record_types
54
+ end
55
+
56
+ def unrecoverable_record_error?(type)
57
+ unrecoverable_record_types.include?(type)
58
+ end
59
+
60
+ def log_os_400_reason(&block)
61
+ if @plugin.log_os_400_reason
62
+ block.call
63
+ else
64
+ @plugin.log.on_debug(&block)
65
+ end
66
+ end
67
+
68
+ def emit_error_label_event?
69
+ !!@plugin.emit_error_label_event
70
+ end
71
+
72
+ def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
73
+ items = response['items']
74
+ if items.nil? || !items.is_a?(Array)
75
+ raise OpenSearchVersionMismatch, "The response format was unrecognized: #{response}"
76
+ end
77
+ if bulk_message_count != items.length
78
+ raise OpenSearchSubmitMismatch, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
79
+ end
80
+ retry_stream = Fluent::MultiEventStream.new
81
+ stats = Hash.new(0)
82
+ meta = {}
83
+ header = {}
84
+ affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
85
+ chunk.msgpack_each do |time, rawrecord|
86
+ bulk_message = ''
87
+ next unless rawrecord.is_a? Hash
88
+ begin
89
+ # we need a deep copy for process_message to alter
90
+ processrecord = Marshal.load(Marshal.dump(rawrecord))
91
+ meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
92
+ next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
93
+ rescue => e
94
+ @plugin.log.debug("Exception in error handler during deep copy: #{e}")
95
+ stats[:bad_chunk_record] += 1
96
+ next
97
+ end
98
+ item = items.shift
99
+ if item.is_a?(Hash) && item.has_key?(@plugin.write_operation)
100
+ write_operation = @plugin.write_operation
101
+ elsif INDEX_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(CREATE_OP)
102
+ write_operation = CREATE_OP
103
+ elsif UPSERT_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(UPDATE_OP)
104
+ write_operation = UPDATE_OP
105
+ elsif item.nil?
106
+ stats[:errors_nil_resp] += 1
107
+ next
108
+ else
109
+ # When we don't have an expected ops field, something changed in the API
110
+ # expected return values.
111
+ stats[:errors_bad_resp] += 1
112
+ next
113
+ end
114
+ if item[write_operation].has_key?('status')
115
+ status = item[write_operation]['status']
116
+ else
117
+ # When we don't have a status field, something changed in the API
118
+ # expected return values.
119
+ stats[:errors_bad_resp] += 1
120
+ next
121
+ end
122
+ case
123
+ when [200, 201].include?(status)
124
+ stats[:successes] += 1
125
+ when CREATE_OP == write_operation && 409 == status
126
+ stats[:duplicates] += 1
127
+ when 400 == status
128
+ stats[:bad_argument] += 1
129
+ reason = ""
130
+ log_os_400_reason do
131
+ if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
132
+ reason = " [error type]: #{item[write_operation]['error']['type']}"
133
+ end
134
+ if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('reason')
135
+ reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
136
+ end
137
+ end
138
+ if emit_error_label_event?
139
+ @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("400 - Rejected by OpenSearch#{reason}"))
140
+ end
141
+ else
142
+ if item[write_operation]['error'].is_a?(String)
143
+ reason = item[write_operation]['error']
144
+ stats[:errors_block_resp] += 1
145
+ if emit_error_label_event?
146
+ @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{reason}"))
147
+ end
148
+ next
149
+ elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
150
+ type = item[write_operation]['error']['type']
151
+ stats[type] += 1
152
+ if unrecoverable_error?(type)
153
+ raise OpenSearchRequestAbortError, "Rejected OpenSearch due to #{type}"
154
+ end
155
+ if unrecoverable_record_error?(type)
156
+ if emit_error_label_event?
157
+ @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{type}: #{reason}"))
158
+ end
159
+ next
160
+ else
161
+ retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
162
+ end
163
+ else
164
+ # When we don't have a type field, something changed in the API
165
+ # expected return values.
166
+ stats[:errors_bad_resp] += 1
167
+ if emit_error_label_event?
168
+ @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - No error type provided in the response"))
169
+ end
170
+ next
171
+ end
172
+ stats[type] += 1
173
+ end
174
+ end
175
+ @plugin.log.on_debug do
176
+ msg = ["Indexed (op = #{@plugin.write_operation})"]
177
+ stats.each_pair { |key, value| msg << "#{value} #{key}" }
178
+ @plugin.log.debug msg.join(', ')
179
+ end
180
+ raise Fluent::Plugin::OpenSearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
181
+ end
182
+ end
@@ -0,0 +1,36 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+
28
+ require 'opensearch/transport/transport/connections/selector'
29
+
30
+ class Fluent::Plugin::OpenSearchFallbackSelector
31
+ include OpenSearch::Transport::Transport::Connections::Selector::Base
32
+
33
+ def select(options={})
34
+ connections.first
35
+ end
36
+ end
@@ -0,0 +1,155 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'fluent/error'
28
+ require_relative './opensearch_error'
29
+
30
+ module Fluent::OpenSearchIndexTemplate
31
+ def get_template(template_file)
32
+ if !File.exist?(template_file)
33
+ raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
34
+ end
35
+ file_contents = IO.read(template_file).gsub(/\n/,'')
36
+ JSON.parse(file_contents)
37
+ end
38
+
39
+ def get_custom_template(template_file, customize_template)
40
+ if !File.exist?(template_file)
41
+ raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
42
+ end
43
+ file_contents = IO.read(template_file).gsub(/\n/,'')
44
+ customize_template.each do |key, value|
45
+ file_contents = file_contents.gsub(key,value.downcase)
46
+ end
47
+ JSON.parse(file_contents)
48
+ end
49
+
50
+ def template_exists?(name, host = nil)
51
+ if @use_legacy_template
52
+ client(host).indices.get_template(:name => name)
53
+ else
54
+ client(host).indices.get_index_template(:name => name)
55
+ end
56
+ return true
57
+ rescue OpenSearch::Transport::Transport::Errors::NotFound
58
+ return false
59
+ end
60
+
61
+ def host_unreachable_exceptions
62
+ client.transport.transport.host_unreachable_exceptions
63
+ end
64
+
65
+ def retry_operate(max_retries, fail_on_retry_exceed = true, catch_transport_exceptions = true)
66
+ return unless block_given?
67
+ retries = 0
68
+ transport_errors = OpenSearch::Transport::Transport::Errors.constants.map{ |c| OpenSearch::Transport::Transport::Errors.const_get c } if catch_transport_exceptions
69
+ begin
70
+ yield
71
+ rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
72
+ @_es = nil
73
+ @_es_info = nil
74
+ if retries < max_retries
75
+ retries += 1
76
+ wait_seconds = 2**retries
77
+ sleep wait_seconds
78
+ log.warn "Could not communicate to OpenSearch, resetting connection and trying again. #{e.message}"
79
+ log.warn "Remaining retry: #{max_retries - retries}. Retry to communicate after #{wait_seconds} second(s)."
80
+ retry
81
+ end
82
+ message = "Could not communicate to OpenSearch after #{retries} retries. #{e.message}"
83
+ log.warn message
84
+ raise Fluent::Plugin::OpenSearchError::RetryableOperationExhaustedFailure,
85
+ message if fail_on_retry_exceed
86
+ end
87
+ end
88
+
89
+ def template_put(name, template, host = nil)
90
+ if @use_legacy_template
91
+ client(host).indices.put_template(:name => name, :body => template)
92
+ else
93
+ client(host).indices.put_index_template(:name => name, :body => template)
94
+ end
95
+ end
96
+
97
+ def indexcreation(index_name, host = nil)
98
+ client(host).indices.create(:index => index_name)
99
+ rescue OpenSearch::Transport::Transport::Error => e
100
+ if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
101
+ log.debug("Index #{index_name} already exists")
102
+ else
103
+ log.error("Error while index creation - #{index_name}", error: e)
104
+ end
105
+ end
106
+
107
+ def template_install(name, template_file, overwrite, host = nil, target_index = nil, index_separator = '-')
108
+ if overwrite
109
+ template_put(name,
110
+ get_template(template_file), host)
111
+
112
+ log.debug("Template '#{name}' overwritten with #{template_file}.")
113
+ return
114
+ end
115
+ if !template_exists?(name, host)
116
+ template_put(name,
117
+ get_template(template_file), host)
118
+ log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
119
+ else
120
+ log.debug("Template '#{name}' configured and already installed.")
121
+ end
122
+ end
123
+
124
+ def template_custom_install(template_name, template_file, overwrite, customize_template, host, target_index, index_separator)
125
+ custom_template = get_custom_template(template_file, customize_template)
126
+
127
+ if overwrite
128
+ template_put(template_name, custom_template, host)
129
+ log.info("Template '#{template_name}' overwritten with #{template_file}.")
130
+ else
131
+ if !template_exists?(template_name, host)
132
+ template_put(template_name, custom_template, host)
133
+ log.info("Template configured, but no template installed. Installed '#{template_name}' from #{template_file}.")
134
+ else
135
+ log.debug("Template '#{template_name}' configured and already installed.")
136
+ end
137
+ end
138
+ end
139
+
140
+ def templates_hash_install(templates, overwrite)
141
+ templates.each do |key, value|
142
+ template_install(key, value, overwrite)
143
+ end
144
+ end
145
+
146
+ def rollover_alias_payload(rollover_alias)
147
+ {
148
+ 'aliases' => {
149
+ rollover_alias => {
150
+ 'is_write_index' => true
151
+ }
152
+ }
153
+ }
154
+ end
155
+ end
@@ -0,0 +1,36 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'opensearch'
28
+
29
+ class Fluent::Plugin::OpenSearchSimpleSniffer < OpenSearch::Transport::Transport::Sniffer
30
+
31
+ def hosts
32
+ @transport.logger.debug "In Fluent::Plugin::OpenSearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
33
+ @transport.hosts
34
+ end
35
+
36
+ end
@@ -0,0 +1,96 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'openssl'
28
+ require 'fluent/configurable'
29
+ require 'fluent/config/error'
30
+
31
+ module Fluent::Plugin
32
+ module OpenSearchTLS
33
+ SUPPORTED_TLS_VERSIONS = if defined?(OpenSSL::SSL::TLS1_3_VERSION)
34
+ [:TLSv1, :TLSv1_1, :TLSv1_2, :TLSv1_3].freeze
35
+ else
36
+ [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
37
+ end
38
+
39
+ DEFAULT_VERSION = :TLSv1_2
40
+ METHODS_MAP = begin
41
+ # When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
42
+ map = {
43
+ TLSv1: OpenSSL::SSL::TLS1_VERSION,
44
+ TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
45
+ TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
46
+ }
47
+ map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
48
+ USE_TLS_MINMAX_VERSION = true
49
+ map.freeze
50
+ rescue NameError
51
+ map = {
52
+ SSLv23: :SSLv23,
53
+ TLSv1: :TLSv1,
54
+ TLSv1_1: :TLSv1_1,
55
+ TLSv1_2: :TLSv1_2,
56
+ }
57
+ USE_TLS_MINMAX_VERSION = false
58
+ end
59
+ private_constant :METHODS_MAP
60
+
61
+ module OpenSearchTLSParams
62
+ include Fluent::Configurable
63
+
64
+ config_param :ssl_version, :enum, list: Fluent::Plugin::OpenSearchTLS::SUPPORTED_TLS_VERSIONS, default: Fluent::Plugin::OpenSearchTLS::DEFAULT_VERSION
65
+ config_param :ssl_min_version, :enum, list: Fluent::Plugin::OpenSearchTLS::SUPPORTED_TLS_VERSIONS, default: nil
66
+ config_param :ssl_max_version, :enum, list: Fluent::Plugin::OpenSearchTLS::SUPPORTED_TLS_VERSIONS, default: nil
67
+ end
68
+
69
+ def self.included(mod)
70
+ mod.include OpenSearchTLSParams
71
+ end
72
+
73
+ def set_tls_minmax_version_config(ssl_version, ssl_max_version, ssl_min_version)
74
+ if USE_TLS_MINMAX_VERSION
75
+ case
76
+ when ssl_min_version.nil? && ssl_max_version.nil?
77
+ ssl_min_version = METHODS_MAP[:TLSv1_2]
78
+ ssl_max_version = METHODS_MAP[:TLSv1_3]
79
+ when ssl_min_version && ssl_max_version.nil?
80
+ raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
81
+ when ssl_min_version.nil? && ssl_max_version
82
+ raise Fluent::ConfigError, "When you set 'ssl_max_version', must set 'ssl_min_version' together."
83
+ else
84
+ ssl_min_version = METHODS_MAP[ssl_min_version]
85
+ ssl_max_version = METHODS_MAP[ssl_max_version]
86
+ end
87
+
88
+ {max_version: ssl_max_version, min_version: ssl_min_version}
89
+ else
90
+ log.warn "'ssl_min_version' does not have any effect in this environment. Use 'ssl_version' instead." unless ssl_min_version.nil?
91
+ log.warn "'ssl_max_version' does not have any effect in this environment. Use 'ssl_version' instead." unless ssl_max_version.nil?
92
+ {version: ssl_version}
93
+ end
94
+ end
95
+ end
96
+ end