opensearch-transport 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +3 -0
  3. data/.gitignore +17 -0
  4. data/Gemfile +47 -0
  5. data/LICENSE +202 -0
  6. data/README.md +551 -0
  7. data/Rakefile +89 -0
  8. data/lib/opensearch/transport/client.rb +354 -0
  9. data/lib/opensearch/transport/redacted.rb +84 -0
  10. data/lib/opensearch/transport/transport/base.rb +450 -0
  11. data/lib/opensearch/transport/transport/connections/collection.rb +136 -0
  12. data/lib/opensearch/transport/transport/connections/connection.rb +169 -0
  13. data/lib/opensearch/transport/transport/connections/selector.rb +101 -0
  14. data/lib/opensearch/transport/transport/errors.rb +100 -0
  15. data/lib/opensearch/transport/transport/http/curb.rb +140 -0
  16. data/lib/opensearch/transport/transport/http/faraday.rb +101 -0
  17. data/lib/opensearch/transport/transport/http/manticore.rb +188 -0
  18. data/lib/opensearch/transport/transport/loggable.rb +94 -0
  19. data/lib/opensearch/transport/transport/response.rb +46 -0
  20. data/lib/opensearch/transport/transport/serializer/multi_json.rb +62 -0
  21. data/lib/opensearch/transport/transport/sniffer.rb +111 -0
  22. data/lib/opensearch/transport/version.rb +31 -0
  23. data/lib/opensearch/transport.rb +46 -0
  24. data/lib/opensearch-transport.rb +27 -0
  25. data/opensearch-transport.gemspec +92 -0
  26. data/spec/opensearch/connections/collection_spec.rb +275 -0
  27. data/spec/opensearch/connections/selector_spec.rb +183 -0
  28. data/spec/opensearch/transport/base_spec.rb +313 -0
  29. data/spec/opensearch/transport/client_spec.rb +1818 -0
  30. data/spec/opensearch/transport/sniffer_spec.rb +284 -0
  31. data/spec/spec_helper.rb +99 -0
  32. data/test/integration/transport_test.rb +108 -0
  33. data/test/profile/client_benchmark_test.rb +141 -0
  34. data/test/test_helper.rb +97 -0
  35. data/test/unit/connection_test.rb +145 -0
  36. data/test/unit/response_test.rb +41 -0
  37. data/test/unit/serializer_test.rb +42 -0
  38. data/test/unit/transport_base_test.rb +673 -0
  39. data/test/unit/transport_curb_test.rb +143 -0
  40. data/test/unit/transport_faraday_test.rb +237 -0
  41. data/test/unit/transport_manticore_test.rb +191 -0
  42. data.tar.gz.sig +1 -0
  43. metadata +456 -0
  44. metadata.gz.sig +1 -0
@@ -0,0 +1,354 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The OpenSearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright OpenSearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Elasticsearch B.V. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Elasticsearch B.V. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ require 'base64'
28
+
29
+ module OpenSearch
30
+ module Transport
31
+ # Handles communication with an OpenSearch cluster.
32
+ #
33
+ # See {file:README.md README} for usage and code examples.
34
+ #
35
+ class Client
36
+ DEFAULT_TRANSPORT_CLASS = Transport::HTTP::Faraday
37
+
38
+ DEFAULT_LOGGER = lambda do
39
+ require 'logger'
40
+ logger = Logger.new(STDERR)
41
+ logger.progname = 'opensearch'
42
+ logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime}: #{msg}\n" }
43
+ logger
44
+ end
45
+
46
+ DEFAULT_TRACER = lambda do
47
+ require 'logger'
48
+ logger = Logger.new(STDERR)
49
+ logger.progname = 'opensearch.tracer'
50
+ logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
51
+ logger
52
+ end
53
+
54
+ # The default host and port to use if not otherwise specified.
55
+ #
56
+ # @since 7.0.0
57
+ DEFAULT_HOST = 'localhost:9200'.freeze
58
+
59
+ # The default port to use if connecting using a Cloud ID.
60
+ # Updated from 9243 to 443 in client version 7.10.1
61
+ #
62
+ # @since 7.2.0
63
+ DEFAULT_CLOUD_PORT = 443
64
+
65
+ # The default port to use if not otherwise specified.
66
+ #
67
+ # @since 7.2.0
68
+ DEFAULT_PORT = 9200
69
+
70
+ # Returns the transport object.
71
+ #
72
+ # @see OpenSearch::Transport::Transport::Base
73
+ # @see OpenSearch::Transport::Transport::HTTP::Faraday
74
+ #
75
+ attr_accessor :transport
76
+
77
+ # Create a client connected to an OpenSearch cluster.
78
+ #
79
+ # Specify the URL via arguments or set the `OPENSEARCH_URL` environment variable.
80
+ #
81
+ # @option arguments [String,Array] :hosts Single host passed as a String or Hash, or multiple hosts
82
+ # passed as an Array; `host` or `url` keys are also valid
83
+ #
84
+ # @option arguments [Boolean] :log Use the default logger (disabled by default)
85
+ #
86
+ # @option arguments [Boolean] :trace Use the default tracer (disabled by default)
87
+ #
88
+ # @option arguments [Object] :logger An instance of a Logger-compatible object
89
+ #
90
+ # @option arguments [Object] :tracer An instance of a Logger-compatible object
91
+ #
92
+ # @option arguments [Number] :resurrect_after After how many seconds a dead connection should be tried again
93
+ #
94
+ # @option arguments [Boolean,Number] :reload_connections Reload connections after X requests (false by default)
95
+ #
96
+ # @option arguments [Boolean] :randomize_hosts Shuffle connections on initialization and reload (false by default)
97
+ #
98
+ # @option arguments [Integer] :sniffer_timeout Timeout for reloading connections in seconds (1 by default)
99
+ #
100
+ # @option arguments [Boolean,Number] :retry_on_failure Retry X times when request fails before raising and
101
+ # exception (false by default)
102
+ # @option arguments Array<Number> :retry_on_status Retry when specific status codes are returned
103
+ #
104
+ # @option arguments [Boolean] :reload_on_failure Reload connections after failure (false by default)
105
+ #
106
+ # @option arguments [Integer] :request_timeout The request timeout to be passed to transport in options
107
+ #
108
+ # @option arguments [Symbol] :adapter A specific adapter for Faraday (e.g. `:patron`)
109
+ #
110
+ # @option arguments [Hash] :transport_options Options to be passed to the `Faraday::Connection` constructor
111
+ #
112
+ # @option arguments [Constant] :transport_class A specific transport class to use, will be initialized by
113
+ # the client and passed hosts and all arguments
114
+ #
115
+ # @option arguments [Object] :transport A specific transport instance
116
+ #
117
+ # @option arguments [Constant] :serializer_class A specific serializer class to use, will be initialized by
118
+ # the transport and passed the transport instance
119
+ #
120
+ # @option arguments [Constant] :selector An instance of selector strategy implemented with
121
+ # {OpenSearch::Transport::Transport::Connections::Selector::Base}.
122
+ #
123
+ # @option arguments [String] :send_get_body_as Specify the HTTP method to use for GET requests with a body.
124
+ # (Default: GET)
125
+ # @option arguments [true, false] :compression Whether to compress requests. Gzip compression will be used.
126
+ # The default is false. Responses will automatically be inflated if they are compressed.
127
+ # If a custom transport object is used, it must handle the request compression and response inflation.
128
+ #
129
+ # @option api_key [String, Hash] :api_key Use API Key Authentication, either the base64 encoding of `id` and `api_key`
130
+ # joined by a colon as a String, or a hash with the `id` and `api_key` values.
131
+ # @option opaque_id_prefix [String] :opaque_id_prefix set a prefix for X-Opaque-Id when initializing the client.
132
+ # This will be prepended to the id you set before each request
133
+ # if you're using X-Opaque-Id
134
+ #
135
+ # @yield [faraday] Access and configure the `Faraday::Connection` instance directly with a block
136
+ #
137
+ def initialize(arguments={}, &block)
138
+ @options = arguments.each_with_object({}){ |(k,v), args| args[k.to_sym] = v }
139
+ @arguments = @options
140
+ @arguments[:logger] ||= @arguments[:log] ? DEFAULT_LOGGER.call() : nil
141
+ @arguments[:tracer] ||= @arguments[:trace] ? DEFAULT_TRACER.call() : nil
142
+ @arguments[:reload_connections] ||= false
143
+ @arguments[:retry_on_failure] ||= false
144
+ @arguments[:reload_on_failure] ||= false
145
+ @arguments[:randomize_hosts] ||= false
146
+ @arguments[:transport_options] ||= {}
147
+ @arguments[:http] ||= {}
148
+ @options[:http] ||= {}
149
+
150
+ set_api_key if (@api_key = @arguments[:api_key])
151
+ set_compatibility_header if ENV['ELASTIC_CLIENT_APIVERSIONING']
152
+
153
+ @seeds = extract_cloud_creds(@arguments)
154
+ @seeds ||= __extract_hosts(@arguments[:hosts] ||
155
+ @arguments[:host] ||
156
+ @arguments[:url] ||
157
+ @arguments[:urls] ||
158
+ ENV['OPENSEARCH_URL'] ||
159
+ DEFAULT_HOST)
160
+
161
+ @send_get_body_as = @arguments[:send_get_body_as] || 'GET'
162
+ @opaque_id_prefix = @arguments[:opaque_id_prefix] || nil
163
+
164
+ if @arguments[:request_timeout]
165
+ @arguments[:transport_options][:request] = { timeout: @arguments[:request_timeout] }
166
+ end
167
+
168
+ if @arguments[:transport]
169
+ @transport = @arguments[:transport]
170
+ else
171
+ @transport_class = @arguments[:transport_class] || DEFAULT_TRANSPORT_CLASS
172
+ @transport = if @transport_class == Transport::HTTP::Faraday
173
+ @arguments[:adapter] ||= __auto_detect_adapter
174
+ @transport_class.new(hosts: @seeds, options: @arguments) do |faraday|
175
+ faraday.adapter(@arguments[:adapter])
176
+ block&.call faraday
177
+ end
178
+ else
179
+ @transport_class.new(hosts: @seeds, options: @arguments)
180
+ end
181
+ end
182
+ end
183
+
184
+ # Performs a request through delegation to {#transport}.
185
+ #
186
+ def perform_request(method, path, params = {}, body = nil, headers = nil)
187
+ method = @send_get_body_as if 'GET' == method && body
188
+ if (opaque_id = params.delete(:opaque_id))
189
+ headers = {} if headers.nil?
190
+ opaque_id = @opaque_id_prefix ? "#{@opaque_id_prefix}#{opaque_id}" : opaque_id
191
+ headers.merge!('X-Opaque-Id' => opaque_id)
192
+ end
193
+ transport.perform_request(method, path, params, body, headers)
194
+ end
195
+
196
+ private
197
+
198
+ def set_api_key
199
+ @api_key = __encode(@api_key) if @api_key.is_a? Hash
200
+ add_header('Authorization' => "ApiKey #{@api_key}")
201
+ @arguments.delete(:user)
202
+ @arguments.delete(:password)
203
+ end
204
+
205
+ def set_compatibility_header
206
+ return unless ['1', 'true'].include?(ENV['ELASTIC_CLIENT_APIVERSIONING'])
207
+
208
+ add_header(
209
+ {
210
+ 'Accept' => 'application/vnd.opensearch+json;compatible-with=7',
211
+ 'Content-Type' => 'application/vnd.opensearch+json; compatible-with=7'
212
+ }
213
+ )
214
+ end
215
+
216
+ def add_header(header)
217
+ headers = @arguments[:transport_options]&.[](:headers) || {}
218
+ headers.merge!(header)
219
+ @arguments[:transport_options].merge!(
220
+ headers: headers
221
+ )
222
+ end
223
+
224
+ def extract_cloud_creds(arguments)
225
+ return unless arguments[:cloud_id] && !arguments[:cloud_id].empty?
226
+
227
+ name = arguments[:cloud_id].split(':')[0]
228
+ cloud_url, opensearch_instance = Base64.decode64(arguments[:cloud_id].gsub("#{name}:", '')).split('$')
229
+
230
+ if cloud_url.include?(':')
231
+ url, port = cloud_url.split(':')
232
+ host = "#{opensearch_instance}.#{url}"
233
+ else
234
+ host = "#{opensearch_instance}.#{cloud_url}"
235
+ port = arguments[:port] || DEFAULT_CLOUD_PORT
236
+ end
237
+ [
238
+ {
239
+ scheme: 'https',
240
+ user: arguments[:user],
241
+ password: arguments[:password],
242
+ host: host,
243
+ port: port.to_i
244
+ }
245
+ ]
246
+ end
247
+
248
+ # Normalizes and returns hosts configuration.
249
+ #
250
+ # Arrayifies the `hosts_config` argument and extracts `host` and `port` info from strings.
251
+ # Performs shuffling when the `randomize_hosts` option is set.
252
+ #
253
+ # TODO: Refactor, so it's available in OpenSearch::Transport::Base as well
254
+ #
255
+ # @return [Array<Hash>]
256
+ # @raise [ArgumentError]
257
+ #
258
+ # @api private
259
+ #
260
+ def __extract_hosts(hosts_config)
261
+ hosts = case hosts_config
262
+ when String
263
+ hosts_config.split(',').map { |h| h.strip! || h }
264
+ when Array
265
+ hosts_config
266
+ when Hash, URI
267
+ [ hosts_config ]
268
+ else
269
+ Array(hosts_config)
270
+ end
271
+
272
+ host_list = hosts.map { |host| __parse_host(host) }
273
+ @options[:randomize_hosts] ? host_list.shuffle! : host_list
274
+ end
275
+
276
+ def __parse_host(host)
277
+ host_parts = case host
278
+ when String
279
+ if host =~ /^[a-z]+\:\/\//
280
+ # Construct a new `URI::Generic` directly from the array returned by URI::split.
281
+ # This avoids `URI::HTTP` and `URI::HTTPS`, which supply default ports.
282
+ uri = URI::Generic.new(*URI.split(host))
283
+ default_port = uri.scheme == 'https' ? 443 : DEFAULT_PORT
284
+ {
285
+ scheme: uri.scheme,
286
+ user: uri.user,
287
+ password: uri.password,
288
+ host: uri.host,
289
+ path: uri.path,
290
+ port: uri.port || default_port
291
+ }
292
+ else
293
+ host, port = host.split(':')
294
+ { host: host, port: port }
295
+ end
296
+ when URI
297
+ {
298
+ scheme: host.scheme,
299
+ user: host.user,
300
+ password: host.password,
301
+ host: host.host,
302
+ path: host.path,
303
+ port: host.port
304
+ }
305
+ when Hash
306
+ host
307
+ else
308
+ raise ArgumentError, "Please pass host as a String, URI or Hash -- #{host.class} given."
309
+ end
310
+ if @api_key
311
+ # Remove Basic Auth if using API KEY
312
+ host_parts.delete(:user)
313
+ host_parts.delete(:password)
314
+ else
315
+ @options[:http][:user] ||= host_parts[:user]
316
+ @options[:http][:password] ||= host_parts[:password]
317
+ end
318
+
319
+ host_parts[:port] = host_parts[:port].to_i if host_parts[:port]
320
+ host_parts[:path].chomp!('/') if host_parts[:path]
321
+ host_parts
322
+ end
323
+
324
+ # Auto-detect the best adapter (HTTP "driver") available, based on libraries
325
+ # loaded by the user, preferring those with persistent connections
326
+ # ("keep-alive") by default
327
+ #
328
+ # @return [Symbol]
329
+ #
330
+ # @api private
331
+ #
332
+ def __auto_detect_adapter
333
+ case
334
+ when defined?(::Patron)
335
+ :patron
336
+ when defined?(::Typhoeus)
337
+ :typhoeus
338
+ when defined?(::HTTPClient)
339
+ :httpclient
340
+ when defined?(::Net::HTTP::Persistent)
341
+ :net_http_persistent
342
+ else
343
+ ::Faraday.default_adapter
344
+ end
345
+ end
346
+
347
+ # Encode credentials for the Authorization Header
348
+ # Credentials is the base64 encoding of id and api_key joined by a colon
349
+ def __encode(api_key)
350
+ Base64.strict_encode64([api_key[:id], api_key[:api_key]].join(':'))
351
+ end
352
+ end
353
+ end
354
+ end
@@ -0,0 +1,84 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # The OpenSearch Contributors require contributions made to
4
+ # this file be licensed under the Apache-2.0 license or a
5
+ # compatible open source license.
6
+ #
7
+ # Modifications Copyright OpenSearch Contributors. See
8
+ # GitHub history for details.
9
+ #
10
+ # Licensed to Elasticsearch B.V. under one or more contributor
11
+ # license agreements. See the NOTICE file distributed with
12
+ # this work for additional information regarding copyright
13
+ # ownership. Elasticsearch B.V. licenses this file to you under
14
+ # the Apache License, Version 2.0 (the "License"); you may
15
+ # not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing,
21
+ # software distributed under the License is distributed on an
22
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
+ # KIND, either express or implied. See the License for the
24
+ # specific language governing permissions and limitations
25
+ # under the License.
26
+
27
+ module OpenSearch
28
+ module Transport
29
+
30
+ # Class for wrapping a hash that could have sensitive data.
31
+ # When printed, the sensitive values will be redacted.
32
+ #
33
+ # @since 6.1.1
34
+ class Redacted < ::Hash
35
+
36
+ def initialize(elements = nil)
37
+ super()
38
+ (elements || {}).each_pair{ |key, value| self[key] = value }
39
+ end
40
+
41
+ # The keys whose values will be redacted.
42
+ #
43
+ # @since 6.1.1
44
+ SENSITIVE_KEYS = [ :password,
45
+ :pwd ].freeze
46
+
47
+ # The replacement string used in place of the value for sensitive keys.
48
+ #
49
+ # @since 6.1.1
50
+ STRING_REPLACEMENT = '<REDACTED>'.freeze
51
+
52
+ # Get a string representation of the hash.
53
+ #
54
+ # @return [ String ] The string representation of the hash.
55
+ #
56
+ # @since 6.1.1
57
+ def inspect
58
+ redacted_string(:inspect)
59
+ end
60
+
61
+ # Get a string representation of the hash.
62
+ #
63
+ # @return [ String ] The string representation of the hash.
64
+ #
65
+ # @since 6.1.1
66
+ def to_s
67
+ redacted_string(:to_s)
68
+ end
69
+
70
+ private
71
+
72
+ def redacted_string(method)
73
+ '{' + reduce([]) do |list, (k, v)|
74
+ list << "#{k.send(method)}=>#{redact(k, v, method)}"
75
+ end.join(', ') + '}'
76
+ end
77
+
78
+ def redact(k, v, method)
79
+ return STRING_REPLACEMENT if SENSITIVE_KEYS.include?(k.to_sym)
80
+ v.send(method)
81
+ end
82
+ end
83
+ end
84
+ end