elastic-transport 8.0.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +7 -0
  2. data/.github/check_license_headers.rb +33 -0
  3. data/.github/license-header.txt +16 -0
  4. data/.github/workflows/license.yml +13 -0
  5. data/.github/workflows/tests.yml +45 -0
  6. data/.gitignore +19 -0
  7. data/CHANGELOG.md +224 -0
  8. data/Gemfile +38 -0
  9. data/LICENSE +202 -0
  10. data/README.md +552 -0
  11. data/Rakefile +87 -0
  12. data/elastic-transport.gemspec +74 -0
  13. data/lib/elastic/transport/client.rb +276 -0
  14. data/lib/elastic/transport/meta_header.rb +135 -0
  15. data/lib/elastic/transport/redacted.rb +73 -0
  16. data/lib/elastic/transport/transport/base.rb +450 -0
  17. data/lib/elastic/transport/transport/connections/collection.rb +126 -0
  18. data/lib/elastic/transport/transport/connections/connection.rb +160 -0
  19. data/lib/elastic/transport/transport/connections/selector.rb +91 -0
  20. data/lib/elastic/transport/transport/errors.rb +91 -0
  21. data/lib/elastic/transport/transport/http/curb.rb +120 -0
  22. data/lib/elastic/transport/transport/http/faraday.rb +95 -0
  23. data/lib/elastic/transport/transport/http/manticore.rb +179 -0
  24. data/lib/elastic/transport/transport/loggable.rb +83 -0
  25. data/lib/elastic/transport/transport/response.rb +36 -0
  26. data/lib/elastic/transport/transport/serializer/multi_json.rb +52 -0
  27. data/lib/elastic/transport/transport/sniffer.rb +101 -0
  28. data/lib/elastic/transport/version.rb +22 -0
  29. data/lib/elastic/transport.rb +37 -0
  30. data/lib/elastic-transport.rb +18 -0
  31. data/spec/elasticsearch/connections/collection_spec.rb +266 -0
  32. data/spec/elasticsearch/connections/selector_spec.rb +166 -0
  33. data/spec/elasticsearch/transport/base_spec.rb +264 -0
  34. data/spec/elasticsearch/transport/client_spec.rb +1651 -0
  35. data/spec/elasticsearch/transport/meta_header_spec.rb +274 -0
  36. data/spec/elasticsearch/transport/sniffer_spec.rb +275 -0
  37. data/spec/spec_helper.rb +90 -0
  38. data/test/integration/transport_test.rb +98 -0
  39. data/test/profile/client_benchmark_test.rb +132 -0
  40. data/test/test_helper.rb +83 -0
  41. data/test/unit/connection_test.rb +135 -0
  42. data/test/unit/response_test.rb +30 -0
  43. data/test/unit/serializer_test.rb +33 -0
  44. data/test/unit/transport_base_test.rb +664 -0
  45. data/test/unit/transport_curb_test.rb +135 -0
  46. data/test/unit/transport_faraday_test.rb +228 -0
  47. data/test/unit/transport_manticore_test.rb +251 -0
  48. metadata +412 -0
@@ -0,0 +1,36 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Elastic
19
+ module Transport
20
+ module Transport
21
+ # Wraps the response from Elasticsearch.
22
+ #
23
+ class Response
24
+ attr_reader :status, :body, :headers
25
+
26
+ # @param status [Integer] Response status code
27
+ # @param body [String] Response body
28
+ # @param headers [Hash] Response headers
29
+ def initialize(status, body, headers={})
30
+ @status, @body, @headers = status, body, headers
31
+ @body = body.force_encoding('UTF-8') if body.respond_to?(:force_encoding)
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,52 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Elastic
19
+ module Transport
20
+ module Transport
21
+ module Serializer
22
+ # An abstract class for implementing serializer implementations
23
+ #
24
+ module Base
25
+ # @param transport [Object] The instance of transport which uses this serializer
26
+ #
27
+ def initialize(transport=nil)
28
+ @transport = transport
29
+ end
30
+ end
31
+
32
+ # A default JSON serializer (using [MultiJSON](http://rubygems.org/gems/multi_json))
33
+ #
34
+ class MultiJson
35
+ include Base
36
+
37
+ # De-serialize a Hash from JSON string
38
+ #
39
+ def load(string, options={})
40
+ ::MultiJson.load(string, options)
41
+ end
42
+
43
+ # Serialize a Hash to JSON string
44
+ #
45
+ def dump(object, options={})
46
+ ::MultiJson.dump(object, options)
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,101 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Elastic
19
+ module Transport
20
+ module Transport
21
+ # Handles node discovery ("sniffing")
22
+ #
23
+ class Sniffer
24
+ PROTOCOL = 'http'
25
+
26
+ attr_reader :transport
27
+ attr_accessor :timeout
28
+
29
+ # @param transport [Object] A transport instance
30
+ #
31
+ def initialize(transport)
32
+ @transport = transport
33
+ @timeout = transport.options[:sniffer_timeout] || 1
34
+ end
35
+
36
+ # Retrieves the node list from the Elasticsearch's
37
+ # [_Nodes Info API_](https://www.elastic.co/guide/reference/api/admin-cluster-nodes-info/)
38
+ # and returns a normalized Array of information suitable for passing to transport.
39
+ #
40
+ # Shuffles the collection before returning it when the `randomize_hosts` option is set for transport.
41
+ #
42
+ # @return [Array<Hash>]
43
+ # @raise [SnifferTimeoutError]
44
+ #
45
+ def hosts
46
+ Timeout::timeout(timeout, SnifferTimeoutError) do
47
+ nodes = perform_sniff_request.body
48
+
49
+ hosts = nodes['nodes'].map do |id, info|
50
+ next unless info[PROTOCOL]
51
+ host, port = parse_publish_address(info[PROTOCOL]['publish_address'])
52
+
53
+ {
54
+ id: id,
55
+ name: info['name'],
56
+ version: info['version'],
57
+ host: host,
58
+ port: port,
59
+ roles: info['roles'],
60
+ attributes: info['attributes']
61
+ }
62
+ end.compact
63
+
64
+ hosts.shuffle! if transport.options[:randomize_hosts]
65
+ hosts
66
+ end
67
+ end
68
+
69
+ private
70
+
71
+ def perform_sniff_request
72
+ transport.perform_request(
73
+ 'GET', '_nodes/http', {}, nil, nil,
74
+ reload_on_failure: false
75
+ )
76
+ end
77
+
78
+ def parse_publish_address(publish_address)
79
+ # publish_address is in the format hostname/ip:port
80
+ if publish_address =~ /\//
81
+ parts = publish_address.partition('/')
82
+ [ parts[0], parse_address_port(parts[2])[1] ]
83
+ else
84
+ parse_address_port(publish_address)
85
+ end
86
+ end
87
+
88
+ def parse_address_port(publish_address)
89
+ # address is ipv6
90
+ if publish_address =~ /[\[\]]/
91
+ if parts = publish_address.match(/\A\[(.+)\](?::(\d+))?\z/)
92
+ [ parts[1], parts[2] ]
93
+ end
94
+ else
95
+ publish_address.split(':')
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,22 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Elastic
19
+ module Transport
20
+ VERSION = '8.0.0.pre1'.freeze
21
+ end
22
+ end
@@ -0,0 +1,37 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ require 'faraday'
19
+ require 'multi_json'
20
+ require 'time'
21
+ require 'timeout'
22
+ require 'uri'
23
+
24
+ require 'elastic/transport/transport/loggable'
25
+ require 'elastic/transport/transport/serializer/multi_json'
26
+ require 'elastic/transport/transport/sniffer'
27
+ require 'elastic/transport/transport/response'
28
+ require 'elastic/transport/transport/errors'
29
+ require 'elastic/transport/transport/base'
30
+ require 'elastic/transport/transport/connections/selector'
31
+ require 'elastic/transport/transport/connections/connection'
32
+ require 'elastic/transport/transport/connections/collection'
33
+ require 'elastic/transport/transport/http/faraday'
34
+ require 'elastic/transport/client'
35
+ require 'elastic/transport/redacted'
36
+
37
+ require 'elastic/transport/version'
@@ -0,0 +1,18 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ require 'elastic/transport'
@@ -0,0 +1,266 @@
1
+ # Licensed to Elasticsearch B.V. under one or more contributor
2
+ # license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright
4
+ # ownership. Elasticsearch B.V. licenses this file to you under
5
+ # the Apache License, Version 2.0 (the "License"); you may
6
+ # not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ require 'spec_helper'
19
+
20
+ describe Elastic::Transport::Transport::Connections::Collection do
21
+ describe '#initialize' do
22
+ let(:collection) do
23
+ described_class.new
24
+ end
25
+
26
+ it 'has an empty list of connections as a default' do
27
+ expect(collection.connections).to be_empty
28
+ end
29
+
30
+ it 'has a default selector class' do
31
+ expect(collection.selector).not_to be_nil
32
+ end
33
+
34
+ context 'when a selector class is specified' do
35
+ let(:collection) do
36
+ described_class.new(selector_class: Elastic::Transport::Transport::Connections::Selector::Random)
37
+ end
38
+
39
+ it 'sets the selector' do
40
+ expect(collection.selector).to be_a(Elastic::Transport::Transport::Connections::Selector::Random)
41
+ end
42
+ end
43
+ end
44
+
45
+ describe '#get_connection' do
46
+ let(:collection) do
47
+ described_class.new(selector_class: Elastic::Transport::Transport::Connections::Selector::Random)
48
+ end
49
+
50
+ before do
51
+ expect(collection.selector).to receive(:select).and_return('OK')
52
+ end
53
+
54
+ it 'uses the selector to select a connection' do
55
+ expect(collection.get_connection).to eq('OK')
56
+ end
57
+ end
58
+
59
+ describe '#hosts' do
60
+ let(:collection) do
61
+ described_class.new(
62
+ connections: [
63
+ double('connection', host: 'A'),
64
+ double('connection', host: 'B')
65
+ ]
66
+ )
67
+ end
68
+
69
+ it 'returns a list of hosts' do
70
+ expect(collection.hosts).to eq([ 'A', 'B'])
71
+ end
72
+ end
73
+
74
+ describe 'enumerable' do
75
+ let(:collection) do
76
+ described_class.new(
77
+ connections: [
78
+ double('connection', host: 'A', dead?: false),
79
+ double('connection', host: 'B', dead?: false)
80
+ ]
81
+ )
82
+ end
83
+
84
+ describe '#map' do
85
+ it 'responds to the method' do
86
+ expect(collection.map { |c| c.host.downcase }).to eq(['a', 'b'])
87
+ end
88
+ end
89
+
90
+ describe '#[]' do
91
+ it 'responds to the method' do
92
+ expect(collection[0].host).to eq('A')
93
+ expect(collection[1].host).to eq('B')
94
+ end
95
+ end
96
+
97
+ describe '#size' do
98
+ it 'responds to the method' do
99
+ expect(collection.size).to eq(2)
100
+ end
101
+ end
102
+
103
+ context 'when a connection is marked as dead' do
104
+ let(:collection) do
105
+ described_class.new(
106
+ connections: [
107
+ double('connection', host: 'A', dead?: true),
108
+ double('connection', host: 'B', dead?: false)
109
+ ]
110
+ )
111
+ end
112
+
113
+ it 'does not enumerate the dead connections' do
114
+ expect(collection.size).to eq(1)
115
+ expect(collection.collect { |c| c.host }).to eq(['B'])
116
+ end
117
+
118
+ context '#alive' do
119
+ it 'enumerates the alive connections' do
120
+ expect(collection.alive.collect { |c| c.host }).to eq(['B'])
121
+ end
122
+ end
123
+
124
+ context '#dead' do
125
+ it 'enumerates the alive connections' do
126
+ expect(collection.dead.collect { |c| c.host }).to eq(['A'])
127
+ end
128
+ end
129
+ end
130
+ end
131
+
132
+ describe '#add' do
133
+
134
+ let(:collection) do
135
+ described_class.new(
136
+ connections: [
137
+ double('connection', host: 'A', dead?: false),
138
+ double('connection', host: 'B', dead?: false)
139
+ ]
140
+ )
141
+ end
142
+
143
+ context 'when an array is provided' do
144
+ before do
145
+ collection.add([double('connection', host: 'C', dead?: false),
146
+ double('connection', host: 'D', dead?: false)])
147
+ end
148
+
149
+ it 'adds the connections' do
150
+ expect(collection.size).to eq(4)
151
+ end
152
+ end
153
+
154
+ context 'when an element is provided' do
155
+ before do
156
+ collection.add(double('connection', host: 'C', dead?: false))
157
+ end
158
+
159
+ it 'adds the connection' do
160
+ expect(collection.size).to eq(3)
161
+ end
162
+ end
163
+ end
164
+
165
+ describe '#remove' do
166
+ let(:connections) do
167
+ [
168
+ double('connection', host: 'A', dead?: false),
169
+ double('connection', host: 'B', dead?: false)
170
+ ]
171
+ end
172
+
173
+ let(:collection) do
174
+ described_class.new(connections: connections)
175
+ end
176
+
177
+ context 'when an array is provided' do
178
+ before do
179
+ collection.remove(connections)
180
+ end
181
+
182
+ it 'removes the connections' do
183
+ expect(collection.size).to eq(0)
184
+ end
185
+ end
186
+
187
+ context 'when an element is provided' do
188
+ let(:connections) do
189
+ [
190
+ double('connection', host: 'A', dead?: false),
191
+ double('connection', host: 'B', dead?: false)
192
+ ]
193
+ end
194
+
195
+ before do
196
+ collection.remove(connections.first)
197
+ end
198
+
199
+ it 'removes the connection' do
200
+ expect(collection.size).to eq(1)
201
+ end
202
+ end
203
+ end
204
+
205
+ describe '#get_connection' do
206
+ context 'when all connections are dead' do
207
+ let(:connection_a) do
208
+ Elastic::Transport::Transport::Connections::Connection.new(host: { host: 'A' })
209
+ end
210
+
211
+ let(:connection_b) do
212
+ Elastic::Transport::Transport::Connections::Connection.new(host: { host: 'B' })
213
+ end
214
+
215
+ let(:collection) do
216
+ described_class.new(connections: [connection_a, connection_b])
217
+ end
218
+
219
+ before do
220
+ connection_a.dead!.dead!
221
+ connection_b.dead!
222
+ end
223
+
224
+ it 'returns the connection with the least failures' do
225
+ expect(collection.get_connection.host[:host]).to eq('B')
226
+ end
227
+ end
228
+
229
+ context 'when multiple threads are used' do
230
+ let(:connections) do
231
+ 20.times.collect do |i|
232
+ Elastic::Transport::Transport::Connections::Connection.new(host: { host: i })
233
+ end
234
+ end
235
+
236
+ let(:collection) do
237
+ described_class.new(connections: connections)
238
+ end
239
+
240
+ it 'allows threads to select connections in parallel' do
241
+ expect(10.times.collect do
242
+ threads = []
243
+ 20.times do
244
+ threads << Thread.new do
245
+ collection.get_connection
246
+ end
247
+ end
248
+ threads.map { |t| t.join }
249
+ collection.get_connection.host[:host]
250
+ end).to eq((0..9).to_a)
251
+ end
252
+
253
+ it 'always returns a connection' do
254
+ threads = 20.times.map do
255
+ Thread.new do
256
+ 20.times.map do
257
+ collection.get_connection.dead!
258
+ end
259
+ end
260
+ end
261
+
262
+ expect(threads.flat_map(&:value).size).to eq(400)
263
+ end
264
+ end
265
+ end
266
+ end