opensearch-transport 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +3 -0
  3. data/.gitignore +17 -0
  4. data/Gemfile +47 -0
  5. data/LICENSE +202 -0
  6. data/README.md +551 -0
  7. data/Rakefile +89 -0
  8. data/lib/opensearch/transport/client.rb +354 -0
  9. data/lib/opensearch/transport/redacted.rb +84 -0
  10. data/lib/opensearch/transport/transport/base.rb +450 -0
  11. data/lib/opensearch/transport/transport/connections/collection.rb +136 -0
  12. data/lib/opensearch/transport/transport/connections/connection.rb +169 -0
  13. data/lib/opensearch/transport/transport/connections/selector.rb +101 -0
  14. data/lib/opensearch/transport/transport/errors.rb +100 -0
  15. data/lib/opensearch/transport/transport/http/curb.rb +140 -0
  16. data/lib/opensearch/transport/transport/http/faraday.rb +101 -0
  17. data/lib/opensearch/transport/transport/http/manticore.rb +188 -0
  18. data/lib/opensearch/transport/transport/loggable.rb +94 -0
  19. data/lib/opensearch/transport/transport/response.rb +46 -0
  20. data/lib/opensearch/transport/transport/serializer/multi_json.rb +62 -0
  21. data/lib/opensearch/transport/transport/sniffer.rb +111 -0
  22. data/lib/opensearch/transport/version.rb +31 -0
  23. data/lib/opensearch/transport.rb +46 -0
  24. data/lib/opensearch-transport.rb +27 -0
  25. data/opensearch-transport.gemspec +92 -0
  26. data/spec/opensearch/connections/collection_spec.rb +275 -0
  27. data/spec/opensearch/connections/selector_spec.rb +183 -0
  28. data/spec/opensearch/transport/base_spec.rb +313 -0
  29. data/spec/opensearch/transport/client_spec.rb +1818 -0
  30. data/spec/opensearch/transport/sniffer_spec.rb +284 -0
  31. data/spec/spec_helper.rb +99 -0
  32. data/test/integration/transport_test.rb +108 -0
  33. data/test/profile/client_benchmark_test.rb +141 -0
  34. data/test/test_helper.rb +97 -0
  35. data/test/unit/connection_test.rb +145 -0
  36. data/test/unit/response_test.rb +41 -0
  37. data/test/unit/serializer_test.rb +42 -0
  38. data/test/unit/transport_base_test.rb +673 -0
  39. data/test/unit/transport_curb_test.rb +143 -0
  40. data/test/unit/transport_faraday_test.rb +237 -0
  41. data/test/unit/transport_manticore_test.rb +191 -0
  42. data.tar.gz.sig +1 -0
  43. metadata +456 -0
  44. metadata.gz.sig +1 -0
@@ -0,0 +1,92 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The OpenSearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright OpenSearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Elasticsearch B.V. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Elasticsearch B.V. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ # coding: utf-8
28
+ lib = File.expand_path('../lib', __FILE__)
29
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
30
+ require 'opensearch/transport/version'
31
+
32
+ signing_key_path = File.expand_path("../gem-private_key.pem")
33
+
34
+ Gem::Specification.new do |s|
35
+ s.name = 'opensearch-transport'
36
+ s.version = OpenSearch::Transport::VERSION
37
+ s.authors = ['Jayesh Hathila', 'Vamshi Vijay Nakkirtha', 'Vijayan Balasubramanian' , 'Yuvraj Jaiswal']
38
+ s.email = ['jayehh@amazon.com', 'vamshin@amazon.com', 'balasvij@amazon.com', 'jaiyuvra@amazon.com']
39
+ s.summary = 'Ruby client for OpenSearch.'
40
+ s.homepage = 'https://opensearch.org/docs/latest/'
41
+ s.license = 'Apache-2.0'
42
+ s.metadata = {
43
+ 'homepage_uri' => 'https://opensearch.org/docs/latest/',
44
+ 'source_code_uri' => 'https://github.com/opensearch-project/opensearch-ruby/tree/main/opensearch-transport',
45
+ 'bug_tracker_uri' => 'https://github.com/opensearch-project/opensearch-ruby/issues'
46
+ }
47
+ s.files = `git ls-files`.split($/)
48
+ s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
49
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
50
+
51
+ if $PROGRAM_NAME.end_with?("gem") && ARGV == ["build", __FILE__] && File.exist?(signing_key_path)
52
+ s.signing_key = signing_key_path
53
+ s.cert_chain = ['../certs/opensearch-rubygems.pem']
54
+ end
55
+
56
+ s.require_paths = ['lib']
57
+
58
+ s.extra_rdoc_files = [ 'README.md', 'LICENSE' ]
59
+ s.rdoc_options = [ '--charset=UTF-8' ]
60
+
61
+ s.required_ruby_version = '>= 2.4'
62
+
63
+ s.add_dependency 'multi_json'
64
+ s.add_dependency 'faraday', '~> 1'
65
+
66
+ s.add_development_dependency 'ansi'
67
+ s.add_development_dependency 'bundler'
68
+ s.add_development_dependency 'cane'
69
+ s.add_development_dependency 'curb' unless defined? JRUBY_VERSION
70
+ s.add_development_dependency 'opensearch-ruby'
71
+ s.add_development_dependency 'hashie'
72
+ s.add_development_dependency 'httpclient'
73
+ s.add_development_dependency 'manticore' if defined? JRUBY_VERSION
74
+ s.add_development_dependency 'minitest'
75
+ s.add_development_dependency 'minitest-reporters'
76
+ s.add_development_dependency 'mocha'
77
+ s.add_development_dependency 'net-http-persistent'
78
+ s.add_development_dependency 'patron' unless defined? JRUBY_VERSION
79
+ s.add_development_dependency 'pry'
80
+ s.add_development_dependency 'rake', '~> 13'
81
+ s.add_development_dependency 'require-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
82
+ s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
83
+ s.add_development_dependency 'shoulda-context'
84
+ s.add_development_dependency 'simplecov'
85
+ s.add_development_dependency 'test-unit', '~> 2'
86
+ s.add_development_dependency 'typhoeus', '~> 1.4'
87
+ s.add_development_dependency 'yard'
88
+
89
+ s.description = <<-DESC.gsub(/^ /, '')
90
+ Ruby client for OpenSearch. See the `opensearch` gem for full integration.
91
+ DESC
92
+ end
@@ -0,0 +1,275 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The OpenSearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright OpenSearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Elasticsearch B.V. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Elasticsearch B.V. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'spec_helper'
28
+
29
+ describe OpenSearch::Transport::Transport::Connections::Collection do
30
+
31
+ describe '#initialize' do
32
+
33
+ let(:collection) do
34
+ described_class.new
35
+ end
36
+
37
+ it 'has an empty list of connections as a default' do
38
+ expect(collection.connections).to be_empty
39
+ end
40
+
41
+ it 'has a default selector class' do
42
+ expect(collection.selector).not_to be_nil
43
+ end
44
+
45
+ context 'when a selector class is specified' do
46
+
47
+ let(:collection) do
48
+ described_class.new(selector_class: OpenSearch::Transport::Transport::Connections::Selector::Random)
49
+ end
50
+
51
+ it 'sets the selector' do
52
+ expect(collection.selector).to be_a(OpenSearch::Transport::Transport::Connections::Selector::Random)
53
+ end
54
+ end
55
+ end
56
+
57
+ describe '#get_connection' do
58
+
59
+ let(:collection) do
60
+ described_class.new(selector_class: OpenSearch::Transport::Transport::Connections::Selector::Random)
61
+ end
62
+
63
+ before do
64
+ expect(collection.selector).to receive(:select).and_return('OK')
65
+ end
66
+
67
+ it 'uses the selector to select a connection' do
68
+ expect(collection.get_connection).to eq('OK')
69
+ end
70
+ end
71
+
72
+ describe '#hosts' do
73
+
74
+ let(:collection) do
75
+ described_class.new(connections: [ double('connection', host: 'A'),
76
+ double('connection', host: 'B') ])
77
+ end
78
+
79
+ it 'returns a list of hosts' do
80
+ expect(collection.hosts).to eq([ 'A', 'B'])
81
+ end
82
+ end
83
+
84
+ describe 'enumerable' do
85
+
86
+ let(:collection) do
87
+ described_class.new(connections: [ double('connection', host: 'A', dead?: false),
88
+ double('connection', host: 'B', dead?: false) ])
89
+ end
90
+
91
+ describe '#map' do
92
+
93
+ it 'responds to the method' do
94
+ expect(collection.map { |c| c.host.downcase }).to eq(['a', 'b'])
95
+ end
96
+ end
97
+
98
+ describe '#[]' do
99
+
100
+ it 'responds to the method' do
101
+ expect(collection[0].host).to eq('A')
102
+ expect(collection[1].host).to eq('B')
103
+ end
104
+ end
105
+
106
+ describe '#size' do
107
+
108
+ it 'responds to the method' do
109
+ expect(collection.size).to eq(2)
110
+ end
111
+ end
112
+
113
+ context 'when a connection is marked as dead' do
114
+
115
+ let(:collection) do
116
+ described_class.new(connections: [ double('connection', host: 'A', dead?: true),
117
+ double('connection', host: 'B', dead?: false) ])
118
+ end
119
+
120
+ it 'does not enumerate the dead connections' do
121
+ expect(collection.size).to eq(1)
122
+ expect(collection.collect { |c| c.host }).to eq(['B'])
123
+ end
124
+
125
+ context '#alive' do
126
+
127
+ it 'enumerates the alive connections' do
128
+ expect(collection.alive.collect { |c| c.host }).to eq(['B'])
129
+ end
130
+ end
131
+
132
+ context '#dead' do
133
+
134
+ it 'enumerates the alive connections' do
135
+ expect(collection.dead.collect { |c| c.host }).to eq(['A'])
136
+ end
137
+ end
138
+ end
139
+ end
140
+
141
+ describe '#add' do
142
+
143
+ let(:collection) do
144
+ described_class.new(connections: [ double('connection', host: 'A', dead?: false),
145
+ double('connection', host: 'B', dead?: false) ])
146
+ end
147
+
148
+ context 'when an array is provided' do
149
+
150
+ before do
151
+ collection.add([double('connection', host: 'C', dead?: false),
152
+ double('connection', host: 'D', dead?: false)])
153
+ end
154
+
155
+ it 'adds the connections' do
156
+ expect(collection.size).to eq(4)
157
+ end
158
+ end
159
+
160
+ context 'when an element is provided' do
161
+
162
+ before do
163
+ collection.add(double('connection', host: 'C', dead?: false))
164
+ end
165
+
166
+ it 'adds the connection' do
167
+ expect(collection.size).to eq(3)
168
+ end
169
+ end
170
+ end
171
+
172
+ describe '#remove' do
173
+
174
+ let(:connections) do
175
+ [ double('connection', host: 'A', dead?: false),
176
+ double('connection', host: 'B', dead?: false) ]
177
+ end
178
+
179
+ let(:collection) do
180
+ described_class.new(connections: connections)
181
+ end
182
+
183
+ context 'when an array is provided' do
184
+
185
+ before do
186
+ collection.remove(connections)
187
+ end
188
+
189
+ it 'removes the connections' do
190
+ expect(collection.size).to eq(0)
191
+ end
192
+ end
193
+
194
+ context 'when an element is provided' do
195
+
196
+ let(:connections) do
197
+ [ double('connection', host: 'A', dead?: false),
198
+ double('connection', host: 'B', dead?: false) ]
199
+ end
200
+
201
+ before do
202
+ collection.remove(connections.first)
203
+ end
204
+
205
+ it 'removes the connection' do
206
+ expect(collection.size).to eq(1)
207
+ end
208
+ end
209
+ end
210
+
211
+ describe '#get_connection' do
212
+
213
+ context 'when all connections are dead' do
214
+
215
+ let(:connection_a) do
216
+ OpenSearch::Transport::Transport::Connections::Connection.new(host: { host: 'A' })
217
+ end
218
+
219
+ let(:connection_b) do
220
+ OpenSearch::Transport::Transport::Connections::Connection.new(host: { host: 'B' })
221
+ end
222
+
223
+ let(:collection) do
224
+ described_class.new(connections: [connection_a, connection_b])
225
+ end
226
+
227
+ before do
228
+ connection_a.dead!.dead!
229
+ connection_b.dead!
230
+ end
231
+
232
+ it 'returns the connection with the least failures' do
233
+ expect(collection.get_connection.host[:host]).to eq('B')
234
+ end
235
+ end
236
+
237
+ context 'when multiple threads are used' do
238
+
239
+ let(:connections) do
240
+ 20.times.collect do |i|
241
+ OpenSearch::Transport::Transport::Connections::Connection.new(host: { host: i })
242
+ end
243
+ end
244
+
245
+ let(:collection) do
246
+ described_class.new(connections: connections)
247
+ end
248
+
249
+ it 'allows threads to select connections in parallel' do
250
+ expect(10.times.collect do
251
+ threads = []
252
+ 20.times do
253
+ threads << Thread.new do
254
+ collection.get_connection
255
+ end
256
+ end
257
+ threads.map { |t| t.join }
258
+ collection.get_connection.host[:host]
259
+ end).to eq((0..9).to_a)
260
+ end
261
+
262
+ it 'always returns a connection' do
263
+ threads = 20.times.map do
264
+ Thread.new do
265
+ 20.times.map do
266
+ collection.get_connection.dead!
267
+ end
268
+ end
269
+ end
270
+
271
+ expect(threads.flat_map(&:value).size).to eq(400)
272
+ end
273
+ end
274
+ end
275
+ end
@@ -0,0 +1,183 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The OpenSearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright OpenSearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Elasticsearch B.V. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Elasticsearch B.V. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'spec_helper'
28
+
29
+ describe OpenSearch::Transport::Transport::Connections::Selector do
30
+
31
+ before do
32
+ class BackupStrategySelector
33
+ include OpenSearch::Transport::Transport::Connections::Selector::Base
34
+
35
+ def select(options={})
36
+ connections.reject do |c|
37
+ c.host[:attributes] && c.host[:attributes][:backup]
38
+ end.sample
39
+ end
40
+ end
41
+ end
42
+
43
+ after do
44
+ Object.send(:remove_const, :BackupStrategySelector)
45
+ end
46
+
47
+ let(:backup_strategy_selector) do
48
+ BackupStrategySelector.new
49
+ end
50
+
51
+ describe 'the Random selector' do
52
+
53
+ let(:connections) do
54
+ [1, 2]
55
+ end
56
+
57
+ let(:selector) do
58
+ described_class::Random.new(connections: connections)
59
+ end
60
+
61
+ it 'is initialized with connections' do
62
+ expect(selector.connections).to eq(connections)
63
+ end
64
+
65
+ describe '#select' do
66
+
67
+ let(:connections) do
68
+ (0..19).to_a
69
+ end
70
+
71
+ it 'returns a connection' do
72
+ expect(selector.select).to be_a(Integer)
73
+ end
74
+
75
+ context 'when multiple threads are used' do
76
+
77
+ it 'allows threads to select connections in parallel' do
78
+ expect(10.times.collect do
79
+ threads = []
80
+ 20.times do
81
+ threads << Thread.new do
82
+ selector.select
83
+ end
84
+ end
85
+ threads.map { |t| t.join }
86
+ selector.select
87
+ end).to all(be_a(Integer))
88
+ end
89
+ end
90
+ end
91
+ end
92
+
93
+ describe 'the RoundRobin selector' do
94
+
95
+ let(:connections) do
96
+ ['A', 'B', 'C']
97
+ end
98
+
99
+ let(:selector) do
100
+ described_class::RoundRobin.new(connections: connections)
101
+ end
102
+
103
+ it 'is initialized with connections' do
104
+ expect(selector.connections).to eq(connections)
105
+ end
106
+
107
+ describe '#select' do
108
+
109
+ it 'rotates over the connections' do
110
+ expect(selector.select).to eq('A')
111
+ expect(selector.select).to eq('B')
112
+ expect(selector.select).to eq('C')
113
+ expect(selector.select).to eq('A')
114
+ end
115
+
116
+ context 'when multiple threads are used' do
117
+
118
+ let(:connections) do
119
+ (0..19).to_a
120
+ end
121
+
122
+ it 'returns a connection' do
123
+ expect(selector.select).to be_a(Integer)
124
+ end
125
+
126
+ it 'allows threads to select connections in parallel' do
127
+ expect(10.times.collect do
128
+ threads = []
129
+ 20.times do
130
+ threads << Thread.new do
131
+ selector.select
132
+ end
133
+ end
134
+ threads.map { |t| t.join }
135
+ selector.select
136
+ end).to eq((0..9).to_a)
137
+ end
138
+ end
139
+ end
140
+ end
141
+
142
+ describe 'a custom selector' do
143
+
144
+ let(:connections) do
145
+ [ double(host: { hostname: 'host1' }),
146
+ double(host: { hostname: 'host2', attributes: { backup: true } }) ]
147
+ end
148
+
149
+ let(:selector) do
150
+ BackupStrategySelector.new(connections: connections)
151
+ end
152
+
153
+ it 'is initialized with connections' do
154
+ expect(selector.connections).to eq(connections)
155
+ end
156
+
157
+ describe '#select' do
158
+
159
+ it 'applies the custom strategy' do
160
+ 10.times { expect(selector.select.host[:hostname]).to eq('host1') }
161
+ end
162
+ end
163
+ end
164
+
165
+ context 'when the Base module is included in a class' do
166
+
167
+ before do
168
+ class ExampleSelector
169
+ include OpenSearch::Transport::Transport::Connections::Selector::Base
170
+ end
171
+ end
172
+
173
+ after do
174
+ Object.send(:remove_const, :ExampleSelector)
175
+ end
176
+
177
+ it 'requires the #select method to be redefined' do
178
+ expect {
179
+ ExampleSelector.new.select
180
+ }.to raise_exception(NoMethodError)
181
+ end
182
+ end
183
+ end