fluent-plugin-input-opensearch 1.1.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +1 -0
  3. data/.editorconfig +9 -0
  4. data/.github/ISSUE_TEMPLATE/bug_report.md +29 -0
  5. data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
  6. data/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +9 -0
  7. data/.github/workflows/coverage.yaml +22 -0
  8. data/.github/workflows/issue-auto-closer.yml +12 -0
  9. data/.github/workflows/linux.yml +26 -0
  10. data/.github/workflows/macos.yml +26 -0
  11. data/.github/workflows/windows.yml +26 -0
  12. data/.gitignore +18 -0
  13. data/CONTRIBUTING.md +24 -0
  14. data/Gemfile +10 -0
  15. data/History.md +67 -0
  16. data/LICENSE.txt +201 -0
  17. data/README.OpenSearchGenID.md +116 -0
  18. data/README.OpenSearchInput.md +314 -0
  19. data/README.Troubleshooting.md +482 -0
  20. data/README.md +1622 -0
  21. data/Rakefile +37 -0
  22. data/fluent-plugin-opensearch.gemspec +39 -0
  23. data/gemfiles/Gemfile.elasticsearch.v6 +12 -0
  24. data/lib/fluent/log-ext.rb +64 -0
  25. data/lib/fluent/plugin/filter_opensearch_genid.rb +103 -0
  26. data/lib/fluent/plugin/in_opensearch.rb +410 -0
  27. data/lib/fluent/plugin/oj_serializer.rb +48 -0
  28. data/lib/fluent/plugin/opensearch_constants.rb +39 -0
  29. data/lib/fluent/plugin/opensearch_error.rb +31 -0
  30. data/lib/fluent/plugin/opensearch_error_handler.rb +182 -0
  31. data/lib/fluent/plugin/opensearch_fallback_selector.rb +36 -0
  32. data/lib/fluent/plugin/opensearch_index_template.rb +155 -0
  33. data/lib/fluent/plugin/opensearch_simple_sniffer.rb +36 -0
  34. data/lib/fluent/plugin/opensearch_tls.rb +96 -0
  35. data/lib/fluent/plugin/out_opensearch.rb +1158 -0
  36. data/lib/fluent/plugin/out_opensearch_data_stream.rb +229 -0
  37. data/test/helper.rb +60 -0
  38. data/test/plugin/datastream_template.json +4 -0
  39. data/test/plugin/test_alias_template.json +9 -0
  40. data/test/plugin/test_filter_opensearch_genid.rb +241 -0
  41. data/test/plugin/test_in_opensearch.rb +500 -0
  42. data/test/plugin/test_index_alias_template.json +11 -0
  43. data/test/plugin/test_index_template.json +25 -0
  44. data/test/plugin/test_oj_serializer.rb +45 -0
  45. data/test/plugin/test_opensearch_error_handler.rb +770 -0
  46. data/test/plugin/test_opensearch_fallback_selector.rb +100 -0
  47. data/test/plugin/test_opensearch_tls.rb +171 -0
  48. data/test/plugin/test_out_opensearch.rb +3980 -0
  49. data/test/plugin/test_out_opensearch_data_stream.rb +746 -0
  50. data/test/plugin/test_template.json +23 -0
  51. data/test/test_log-ext.rb +61 -0
  52. metadata +291 -0
@@ -0,0 +1,100 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require_relative '../helper'
28
+ require 'fluent/test/driver/output'
29
+ require 'fluent/plugin/out_opensearch'
30
+
31
+ class OpenSearchFallbackSelectorTest < Test::Unit::TestCase
32
+ attr_accessor :index_cmds
33
+
34
+ def setup
35
+ Fluent::Test.setup
36
+ @driver = nil
37
+ log = Fluent::Engine.log
38
+ log.out.logs.slice!(0, log.out.logs.length)
39
+ end
40
+
41
+ def stub_opensearch(url="http://localhost:9200/_bulk")
42
+ stub_request(:post, url).with do |req|
43
+ @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
44
+ end
45
+ end
46
+
47
+ def stub_opensearch_info(url="http://localhost:9200/", version="1.2.2")
48
+ body ="{\"version\":{\"number\":\"#{version}\", \"distribution\":\"opensearch\"},\"tagline\":\"The OpenSearch Project: https://opensearch.org/\"}"
49
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
50
+ end
51
+
52
+ def stub_opensearch_info_not_found(url="http://localhost:9200/", version="1.2.2")
53
+ stub_request(:get, url).to_return(:status => [404, "Not Found"])
54
+ end
55
+
56
+ def stub_opensearch_info_unavailable(url="http://localhost:9200/", version="1.2.2")
57
+ stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
58
+ end
59
+
60
+ def sample_record(content={})
61
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
62
+ end
63
+
64
+ def driver(conf='')
65
+ @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::OpenSearchOutput) {
66
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
67
+ if !defined?(Fluent::Plugin::Output)
68
+ def format(tag, time, record)
69
+ [time, record].to_msgpack
70
+ end
71
+ end
72
+ }.configure(conf)
73
+ end
74
+
75
+ def test_fallback_on_info
76
+ stub_opensearch_info_not_found("http://localhost:9202/")
77
+ stub_opensearch_info_unavailable("http://localhost:9201/")
78
+ stub_opensearch_info
79
+ stub_opensearch
80
+ config = %[
81
+ hosts localhost:9202,localhost:9201,localhost:9200
82
+ selector_class_name Fluent::Plugin::OpenSearchFallbackSelector
83
+ @log_level debug
84
+ with_transporter_log true
85
+ reload_connections true
86
+ reload_after 10
87
+ catch_transport_exception_on_retry false # For fallback testing
88
+ ]
89
+ assert_raise(OpenSearch::Transport::Transport::Errors::NotFound) do
90
+ driver(config)
91
+ end
92
+ driver.run(default_tag: 'test') do
93
+ driver.feed(sample_record)
94
+ end
95
+ assert_equal(2, index_cmds.length)
96
+ assert_equal("fluentd", index_cmds.first['index']['_index'])
97
+ end
98
+
99
+ # TODO: on feed phase test case
100
+ end
@@ -0,0 +1,171 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The fluent-plugin-opensearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright fluent-plugin-opensearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Uken Inc. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Uken Inc. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require_relative '../helper'
28
+ require 'fluent/test/driver/output'
29
+ require 'fluent/plugin/output'
30
+ require 'fluent/plugin/opensearch_tls'
31
+
32
+ class TestOpenSearchTLS < Test::Unit::TestCase
33
+
34
+ class TestTLSModuleOutput < Fluent::Plugin::Output
35
+ include Fluent::Plugin::OpenSearchTLS
36
+
37
+ def initialize
38
+ super
39
+ @emit_streams = []
40
+ end
41
+
42
+ def write(chunk)
43
+ es = Fluent::ArrayEventStream.new
44
+ chunk.each do |time, record|
45
+ es.add(time, record)
46
+ end
47
+ @emit_streams << [tag, es]
48
+ end
49
+ end
50
+
51
+ setup do
52
+ Fluent::Test.setup
53
+ @use_tls_minmax_version = begin
54
+ map = {
55
+ TLSv1: OpenSSL::SSL::TLS1_VERSION,
56
+ TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
57
+ TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
58
+ }
59
+ map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
60
+ true
61
+ rescue NameError
62
+ false
63
+ end
64
+ @enabled_tlsv1_3 = begin
65
+ map = {TLSv1_3: OpenSSL::SSL::TLS1_3_VERSION}
66
+ true
67
+ rescue NameError
68
+ false
69
+ end
70
+ end
71
+
72
+ def driver(conf='')
73
+ Fluent::Test::Driver::Output.new(TestTLSModuleOutput).configure(conf)
74
+ end
75
+
76
+ test 'configure' do
77
+ assert_equal Fluent::Plugin::OpenSearchTLS::DEFAULT_VERSION, driver.instance.ssl_version
78
+ assert_nil driver.instance.ssl_max_version
79
+ assert_nil driver.instance.ssl_min_version
80
+ end
81
+
82
+ test 'check USE_TLS_MINMAX_VERSION value' do
83
+ assert_equal @use_tls_minmax_version, Fluent::Plugin::OpenSearchTLS::USE_TLS_MINMAX_VERSION
84
+ end
85
+
86
+ sub_test_case 'set_tls_minmax_version_config' do
87
+ test 'default' do
88
+ d = driver('')
89
+ ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
90
+ if @use_tls_minmax_version
91
+ if @enabled_tlsv1_3
92
+ assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
93
+ min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
94
+ else
95
+ assert_equal({max_version: nil,
96
+ min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
97
+
98
+ end
99
+ else
100
+ assert_equal({version: Fluent::Plugin::OpenSearchTLS::DEFAULT_VERSION}, ssl_version_options)
101
+ end
102
+ end
103
+
104
+ test 'errorous cases' do
105
+ if @use_tls_minmax_version
106
+ assert_raise(Fluent::ConfigError) do
107
+ d = driver(%{ssl_max_version TLSv1_2})
108
+ d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
109
+ d.instance.ssl_max_version,
110
+ d.instance.ssl_min_version)
111
+ end
112
+ assert_raise(Fluent::ConfigError) do
113
+ d = driver(%{ssl_min_version TLSv1_2})
114
+ d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
115
+ d.instance.ssl_max_version,
116
+ d.instance.ssl_min_version)
117
+ end
118
+ else
119
+ d1 = driver(%{
120
+ ssl_max_version TLSv1_2
121
+ @log_level info
122
+ })
123
+ d1.instance.set_tls_minmax_version_config(d1.instance.ssl_version,
124
+ d1.instance.ssl_max_version,
125
+ d1.instance.ssl_min_version)
126
+
127
+ d1.logs.any? {|a| a.include?("'ssl_max_version' does not have any effect in this environment.") }
128
+ d2 = driver(%{
129
+ ssl_min_version TLSv1_2
130
+ @log_level info
131
+ })
132
+ d2.instance.set_tls_minmax_version_config(d2.instance.ssl_version,
133
+ d2.instance.ssl_max_version,
134
+ d2.instance.ssl_min_version)
135
+ d2.logs.any? {|a| a.include?("'ssl_min_version' does not have any effect in this environment.") }
136
+ end
137
+ end
138
+
139
+ test 'min_version & max_version' do
140
+ config = %{
141
+ ssl_max_version TLSv1_2
142
+ ssl_min_version TLSv1_1
143
+ }
144
+ d = driver(config)
145
+ ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
146
+ d.instance.ssl_max_version,
147
+ d.instance.ssl_min_version)
148
+ if @use_tls_minmax_version
149
+ assert_equal({max_version: OpenSSL::SSL::TLS1_2_VERSION,
150
+ min_version: OpenSSL::SSL::TLS1_1_VERSION}, ssl_version_options)
151
+ else
152
+ assert_equal({version: Fluent::Plugin::OpenSearchTLS::DEFAULT_VERSION}, ssl_version_options)
153
+ end
154
+ end
155
+
156
+ test 'TLSv1.3' do
157
+ omit "openssl gem does not support TLSv1.3" unless @enabled_tlsv1_3
158
+ config = %{
159
+ ssl_max_version TLSv1_3
160
+ ssl_min_version TLSv1_2
161
+ }
162
+ d = driver(config)
163
+ ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
164
+ d.instance.ssl_max_version,
165
+ d.instance.ssl_min_version)
166
+ assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
167
+ min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
168
+
169
+ end
170
+ end
171
+ end