fluent-plugin-input-opensearch 1.2.3 → 1.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/fluent/plugin/in_opensearch.rb +8 -6
- data/lib/fluent/plugin/out_opensearch.rb +5 -1
- metadata +5 -55
- data/.coveralls.yml +0 -1
- data/.editorconfig +0 -9
- data/.github/ISSUE_TEMPLATE/bug_report.md +0 -29
- data/.github/ISSUE_TEMPLATE/feature_request.md +0 -24
- data/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +0 -9
- data/.github/workflows/coverage.yaml +0 -22
- data/.github/workflows/issue-auto-closer.yml +0 -12
- data/.github/workflows/linux.yml +0 -26
- data/.github/workflows/macos.yml +0 -26
- data/.github/workflows/windows.yml +0 -26
- data/.gitignore +0 -18
- data/CONTRIBUTING.md +0 -24
- data/Gemfile +0 -10
- data/History.md +0 -67
- data/README.OpenSearchGenID.md +0 -116
- data/README.OpenSearchInput.md +0 -396
- data/README.Troubleshooting.md +0 -482
- data/Rakefile +0 -37
- data/fluent-plugin-opensearch.gemspec +0 -39
- data/gemfiles/Gemfile.elasticsearch.v6 +0 -12
- data/test/helper.rb +0 -60
- data/test/plugin/datastream_template.json +0 -4
- data/test/plugin/test_alias_template.json +0 -9
- data/test/plugin/test_filter_opensearch_genid.rb +0 -241
- data/test/plugin/test_in_opensearch.rb +0 -500
- data/test/plugin/test_index_alias_template.json +0 -11
- data/test/plugin/test_index_template.json +0 -25
- data/test/plugin/test_oj_serializer.rb +0 -45
- data/test/plugin/test_opensearch_error_handler.rb +0 -770
- data/test/plugin/test_opensearch_fallback_selector.rb +0 -100
- data/test/plugin/test_opensearch_tls.rb +0 -171
- data/test/plugin/test_out_opensearch.rb +0 -3980
- data/test/plugin/test_out_opensearch_data_stream.rb +0 -746
- data/test/plugin/test_template.json +0 -23
- data/test/test_log-ext.rb +0 -61
@@ -1,100 +0,0 @@
|
|
1
|
-
# SPDX-License-Identifier: Apache-2.0
|
2
|
-
#
|
3
|
-
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
-
# this file be licensed under the Apache-2.0 license or a
|
5
|
-
# compatible open source license.
|
6
|
-
#
|
7
|
-
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
-
# GitHub history for details.
|
9
|
-
#
|
10
|
-
# Licensed to Uken Inc. under one or more contributor
|
11
|
-
# license agreements. See the NOTICE file distributed with
|
12
|
-
# this work for additional information regarding copyright
|
13
|
-
# ownership. Uken Inc. licenses this file to you under
|
14
|
-
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
-
# not use this file except in compliance with the License.
|
16
|
-
# You may obtain a copy of the License at
|
17
|
-
#
|
18
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
-
#
|
20
|
-
# Unless required by applicable law or agreed to in writing,
|
21
|
-
# software distributed under the License is distributed on an
|
22
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
-
# KIND, either express or implied. See the License for the
|
24
|
-
# specific language governing permissions and limitations
|
25
|
-
# under the License.
|
26
|
-
|
27
|
-
require_relative '../helper'
|
28
|
-
require 'fluent/test/driver/output'
|
29
|
-
require 'fluent/plugin/out_opensearch'
|
30
|
-
|
31
|
-
class OpenSearchFallbackSelectorTest < Test::Unit::TestCase
|
32
|
-
attr_accessor :index_cmds
|
33
|
-
|
34
|
-
def setup
|
35
|
-
Fluent::Test.setup
|
36
|
-
@driver = nil
|
37
|
-
log = Fluent::Engine.log
|
38
|
-
log.out.logs.slice!(0, log.out.logs.length)
|
39
|
-
end
|
40
|
-
|
41
|
-
def stub_opensearch(url="http://localhost:9200/_bulk")
|
42
|
-
stub_request(:post, url).with do |req|
|
43
|
-
@index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def stub_opensearch_info(url="http://localhost:9200/", version="1.2.2")
|
48
|
-
body ="{\"version\":{\"number\":\"#{version}\", \"distribution\":\"opensearch\"},\"tagline\":\"The OpenSearch Project: https://opensearch.org/\"}"
|
49
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
50
|
-
end
|
51
|
-
|
52
|
-
def stub_opensearch_info_not_found(url="http://localhost:9200/", version="1.2.2")
|
53
|
-
stub_request(:get, url).to_return(:status => [404, "Not Found"])
|
54
|
-
end
|
55
|
-
|
56
|
-
def stub_opensearch_info_unavailable(url="http://localhost:9200/", version="1.2.2")
|
57
|
-
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
|
58
|
-
end
|
59
|
-
|
60
|
-
def sample_record(content={})
|
61
|
-
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
|
62
|
-
end
|
63
|
-
|
64
|
-
def driver(conf='')
|
65
|
-
@driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::OpenSearchOutput) {
|
66
|
-
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
67
|
-
if !defined?(Fluent::Plugin::Output)
|
68
|
-
def format(tag, time, record)
|
69
|
-
[time, record].to_msgpack
|
70
|
-
end
|
71
|
-
end
|
72
|
-
}.configure(conf)
|
73
|
-
end
|
74
|
-
|
75
|
-
def test_fallback_on_info
|
76
|
-
stub_opensearch_info_not_found("http://localhost:9202/")
|
77
|
-
stub_opensearch_info_unavailable("http://localhost:9201/")
|
78
|
-
stub_opensearch_info
|
79
|
-
stub_opensearch
|
80
|
-
config = %[
|
81
|
-
hosts localhost:9202,localhost:9201,localhost:9200
|
82
|
-
selector_class_name Fluent::Plugin::OpenSearchFallbackSelector
|
83
|
-
@log_level debug
|
84
|
-
with_transporter_log true
|
85
|
-
reload_connections true
|
86
|
-
reload_after 10
|
87
|
-
catch_transport_exception_on_retry false # For fallback testing
|
88
|
-
]
|
89
|
-
assert_raise(OpenSearch::Transport::Transport::Errors::NotFound) do
|
90
|
-
driver(config)
|
91
|
-
end
|
92
|
-
driver.run(default_tag: 'test') do
|
93
|
-
driver.feed(sample_record)
|
94
|
-
end
|
95
|
-
assert_equal(2, index_cmds.length)
|
96
|
-
assert_equal("fluentd", index_cmds.first['index']['_index'])
|
97
|
-
end
|
98
|
-
|
99
|
-
# TODO: on feed phase test case
|
100
|
-
end
|
@@ -1,171 +0,0 @@
|
|
1
|
-
# SPDX-License-Identifier: Apache-2.0
|
2
|
-
#
|
3
|
-
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
-
# this file be licensed under the Apache-2.0 license or a
|
5
|
-
# compatible open source license.
|
6
|
-
#
|
7
|
-
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
-
# GitHub history for details.
|
9
|
-
#
|
10
|
-
# Licensed to Uken Inc. under one or more contributor
|
11
|
-
# license agreements. See the NOTICE file distributed with
|
12
|
-
# this work for additional information regarding copyright
|
13
|
-
# ownership. Uken Inc. licenses this file to you under
|
14
|
-
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
-
# not use this file except in compliance with the License.
|
16
|
-
# You may obtain a copy of the License at
|
17
|
-
#
|
18
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
-
#
|
20
|
-
# Unless required by applicable law or agreed to in writing,
|
21
|
-
# software distributed under the License is distributed on an
|
22
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
-
# KIND, either express or implied. See the License for the
|
24
|
-
# specific language governing permissions and limitations
|
25
|
-
# under the License.
|
26
|
-
|
27
|
-
require_relative '../helper'
|
28
|
-
require 'fluent/test/driver/output'
|
29
|
-
require 'fluent/plugin/output'
|
30
|
-
require 'fluent/plugin/opensearch_tls'
|
31
|
-
|
32
|
-
class TestOpenSearchTLS < Test::Unit::TestCase
|
33
|
-
|
34
|
-
class TestTLSModuleOutput < Fluent::Plugin::Output
|
35
|
-
include Fluent::Plugin::OpenSearchTLS
|
36
|
-
|
37
|
-
def initialize
|
38
|
-
super
|
39
|
-
@emit_streams = []
|
40
|
-
end
|
41
|
-
|
42
|
-
def write(chunk)
|
43
|
-
es = Fluent::ArrayEventStream.new
|
44
|
-
chunk.each do |time, record|
|
45
|
-
es.add(time, record)
|
46
|
-
end
|
47
|
-
@emit_streams << [tag, es]
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
setup do
|
52
|
-
Fluent::Test.setup
|
53
|
-
@use_tls_minmax_version = begin
|
54
|
-
map = {
|
55
|
-
TLSv1: OpenSSL::SSL::TLS1_VERSION,
|
56
|
-
TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
|
57
|
-
TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
|
58
|
-
}
|
59
|
-
map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
|
60
|
-
true
|
61
|
-
rescue NameError
|
62
|
-
false
|
63
|
-
end
|
64
|
-
@enabled_tlsv1_3 = begin
|
65
|
-
map = {TLSv1_3: OpenSSL::SSL::TLS1_3_VERSION}
|
66
|
-
true
|
67
|
-
rescue NameError
|
68
|
-
false
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
def driver(conf='')
|
73
|
-
Fluent::Test::Driver::Output.new(TestTLSModuleOutput).configure(conf)
|
74
|
-
end
|
75
|
-
|
76
|
-
test 'configure' do
|
77
|
-
assert_equal Fluent::Plugin::OpenSearchTLS::DEFAULT_VERSION, driver.instance.ssl_version
|
78
|
-
assert_nil driver.instance.ssl_max_version
|
79
|
-
assert_nil driver.instance.ssl_min_version
|
80
|
-
end
|
81
|
-
|
82
|
-
test 'check USE_TLS_MINMAX_VERSION value' do
|
83
|
-
assert_equal @use_tls_minmax_version, Fluent::Plugin::OpenSearchTLS::USE_TLS_MINMAX_VERSION
|
84
|
-
end
|
85
|
-
|
86
|
-
sub_test_case 'set_tls_minmax_version_config' do
|
87
|
-
test 'default' do
|
88
|
-
d = driver('')
|
89
|
-
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
|
90
|
-
if @use_tls_minmax_version
|
91
|
-
if @enabled_tlsv1_3
|
92
|
-
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
93
|
-
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
94
|
-
else
|
95
|
-
assert_equal({max_version: nil,
|
96
|
-
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
97
|
-
|
98
|
-
end
|
99
|
-
else
|
100
|
-
assert_equal({version: Fluent::Plugin::OpenSearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
test 'errorous cases' do
|
105
|
-
if @use_tls_minmax_version
|
106
|
-
assert_raise(Fluent::ConfigError) do
|
107
|
-
d = driver(%{ssl_max_version TLSv1_2})
|
108
|
-
d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
|
109
|
-
d.instance.ssl_max_version,
|
110
|
-
d.instance.ssl_min_version)
|
111
|
-
end
|
112
|
-
assert_raise(Fluent::ConfigError) do
|
113
|
-
d = driver(%{ssl_min_version TLSv1_2})
|
114
|
-
d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
|
115
|
-
d.instance.ssl_max_version,
|
116
|
-
d.instance.ssl_min_version)
|
117
|
-
end
|
118
|
-
else
|
119
|
-
d1 = driver(%{
|
120
|
-
ssl_max_version TLSv1_2
|
121
|
-
@log_level info
|
122
|
-
})
|
123
|
-
d1.instance.set_tls_minmax_version_config(d1.instance.ssl_version,
|
124
|
-
d1.instance.ssl_max_version,
|
125
|
-
d1.instance.ssl_min_version)
|
126
|
-
|
127
|
-
d1.logs.any? {|a| a.include?("'ssl_max_version' does not have any effect in this environment.") }
|
128
|
-
d2 = driver(%{
|
129
|
-
ssl_min_version TLSv1_2
|
130
|
-
@log_level info
|
131
|
-
})
|
132
|
-
d2.instance.set_tls_minmax_version_config(d2.instance.ssl_version,
|
133
|
-
d2.instance.ssl_max_version,
|
134
|
-
d2.instance.ssl_min_version)
|
135
|
-
d2.logs.any? {|a| a.include?("'ssl_min_version' does not have any effect in this environment.") }
|
136
|
-
end
|
137
|
-
end
|
138
|
-
|
139
|
-
test 'min_version & max_version' do
|
140
|
-
config = %{
|
141
|
-
ssl_max_version TLSv1_2
|
142
|
-
ssl_min_version TLSv1_1
|
143
|
-
}
|
144
|
-
d = driver(config)
|
145
|
-
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
|
146
|
-
d.instance.ssl_max_version,
|
147
|
-
d.instance.ssl_min_version)
|
148
|
-
if @use_tls_minmax_version
|
149
|
-
assert_equal({max_version: OpenSSL::SSL::TLS1_2_VERSION,
|
150
|
-
min_version: OpenSSL::SSL::TLS1_1_VERSION}, ssl_version_options)
|
151
|
-
else
|
152
|
-
assert_equal({version: Fluent::Plugin::OpenSearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
153
|
-
end
|
154
|
-
end
|
155
|
-
|
156
|
-
test 'TLSv1.3' do
|
157
|
-
omit "openssl gem does not support TLSv1.3" unless @enabled_tlsv1_3
|
158
|
-
config = %{
|
159
|
-
ssl_max_version TLSv1_3
|
160
|
-
ssl_min_version TLSv1_2
|
161
|
-
}
|
162
|
-
d = driver(config)
|
163
|
-
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
|
164
|
-
d.instance.ssl_max_version,
|
165
|
-
d.instance.ssl_min_version)
|
166
|
-
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
167
|
-
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
168
|
-
|
169
|
-
end
|
170
|
-
end
|
171
|
-
end
|