logstash-output-elasticsearch-test 11.16.0-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +649 -0
  3. data/CONTRIBUTORS +34 -0
  4. data/Gemfile +16 -0
  5. data/LICENSE +202 -0
  6. data/NOTICE.TXT +5 -0
  7. data/README.md +106 -0
  8. data/docs/index.asciidoc +1369 -0
  9. data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +282 -0
  10. data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
  11. data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +155 -0
  12. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +534 -0
  13. data/lib/logstash/outputs/elasticsearch/http_client.rb +497 -0
  14. data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +201 -0
  15. data/lib/logstash/outputs/elasticsearch/ilm.rb +92 -0
  16. data/lib/logstash/outputs/elasticsearch/license_checker.rb +52 -0
  17. data/lib/logstash/outputs/elasticsearch/template_manager.rb +131 -0
  18. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json +45 -0
  19. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json +44 -0
  20. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json +50 -0
  21. data/lib/logstash/outputs/elasticsearch.rb +699 -0
  22. data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +237 -0
  23. data/lib/logstash/plugin_mixins/elasticsearch/common.rb +409 -0
  24. data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb +9 -0
  25. data/logstash-output-elasticsearch.gemspec +40 -0
  26. data/spec/es_spec_helper.rb +225 -0
  27. data/spec/fixtures/_nodes/6x.json +81 -0
  28. data/spec/fixtures/_nodes/7x.json +92 -0
  29. data/spec/fixtures/htpasswd +2 -0
  30. data/spec/fixtures/license_check/active.json +16 -0
  31. data/spec/fixtures/license_check/inactive.json +5 -0
  32. data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
  33. data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
  34. data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
  35. data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
  36. data/spec/fixtures/template-with-policy-es6x.json +48 -0
  37. data/spec/fixtures/template-with-policy-es7x.json +45 -0
  38. data/spec/fixtures/template-with-policy-es8x.json +50 -0
  39. data/spec/fixtures/test_certs/ca.crt +29 -0
  40. data/spec/fixtures/test_certs/ca.der.sha256 +1 -0
  41. data/spec/fixtures/test_certs/ca.key +51 -0
  42. data/spec/fixtures/test_certs/renew.sh +13 -0
  43. data/spec/fixtures/test_certs/test.crt +30 -0
  44. data/spec/fixtures/test_certs/test.der.sha256 +1 -0
  45. data/spec/fixtures/test_certs/test.key +51 -0
  46. data/spec/fixtures/test_certs/test.p12 +0 -0
  47. data/spec/fixtures/test_certs/test_invalid.crt +36 -0
  48. data/spec/fixtures/test_certs/test_invalid.key +51 -0
  49. data/spec/fixtures/test_certs/test_invalid.p12 +0 -0
  50. data/spec/fixtures/test_certs/test_self_signed.crt +32 -0
  51. data/spec/fixtures/test_certs/test_self_signed.key +54 -0
  52. data/spec/fixtures/test_certs/test_self_signed.p12 +0 -0
  53. data/spec/integration/outputs/compressed_indexing_spec.rb +70 -0
  54. data/spec/integration/outputs/create_spec.rb +67 -0
  55. data/spec/integration/outputs/data_stream_spec.rb +68 -0
  56. data/spec/integration/outputs/delete_spec.rb +63 -0
  57. data/spec/integration/outputs/ilm_spec.rb +534 -0
  58. data/spec/integration/outputs/index_spec.rb +421 -0
  59. data/spec/integration/outputs/index_version_spec.rb +98 -0
  60. data/spec/integration/outputs/ingest_pipeline_spec.rb +75 -0
  61. data/spec/integration/outputs/metrics_spec.rb +66 -0
  62. data/spec/integration/outputs/no_es_on_startup_spec.rb +78 -0
  63. data/spec/integration/outputs/painless_update_spec.rb +99 -0
  64. data/spec/integration/outputs/parent_spec.rb +94 -0
  65. data/spec/integration/outputs/retry_spec.rb +182 -0
  66. data/spec/integration/outputs/routing_spec.rb +61 -0
  67. data/spec/integration/outputs/sniffer_spec.rb +94 -0
  68. data/spec/integration/outputs/templates_spec.rb +133 -0
  69. data/spec/integration/outputs/unsupported_actions_spec.rb +75 -0
  70. data/spec/integration/outputs/update_spec.rb +114 -0
  71. data/spec/spec_helper.rb +10 -0
  72. data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
  73. data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
  74. data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
  75. data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
  76. data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
  77. data/spec/unit/http_client_builder_spec.rb +185 -0
  78. data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +612 -0
  79. data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +151 -0
  80. data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +501 -0
  81. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +339 -0
  82. data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +189 -0
  83. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +103 -0
  84. data/spec/unit/outputs/elasticsearch_spec.rb +1573 -0
  85. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +197 -0
  86. data/spec/unit/outputs/error_whitelist_spec.rb +56 -0
  87. data/spec/unit/outputs/license_check_spec.rb +57 -0
  88. metadata +423 -0
@@ -0,0 +1,151 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/elasticsearch/http_client"
3
+ require 'cabin'
4
+
5
+ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
6
+ let(:logger) { Cabin::Channel.get }
7
+ let(:options) { {} }
8
+
9
+ subject { described_class.new(logger, options) }
10
+
11
+ it "should raise an exception if requests are issued after close" do
12
+ subject.close
13
+ begin
14
+ subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/')
15
+ fail 'expected to raise a HostUnreachableError'
16
+ rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
17
+ expect( e.original_error ).to be_a ::Manticore::ClientStoppedException
18
+ end
19
+ end
20
+
21
+ describe "auth" do
22
+ let(:user) { "myuser" }
23
+ let(:password) { "mypassword" }
24
+ let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
25
+ let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
26
+
27
+ it "should convert the auth to params" do
28
+ resp = double("response")
29
+ allow(resp).to receive(:call)
30
+ allow(resp).to receive(:code).and_return(200)
31
+
32
+ expected_uri = noauth_uri.clone
33
+ expected_uri.path = "/"
34
+
35
+ expect(subject.manticore).to receive(:get).
36
+ with(expected_uri.to_s, {
37
+ :headers => {"Content-Type" => "application/json"},
38
+ :auth => {
39
+ :user => user,
40
+ :password => password,
41
+ :eager => true
42
+ }
43
+ }).and_return resp
44
+
45
+ subject.perform_request(uri, :get, "/")
46
+ end
47
+ end
48
+
49
+ describe "bad response codes" do
50
+ let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
51
+
52
+ it "should raise a bad response code error" do
53
+ resp = double("response")
54
+ allow(resp).to receive(:call)
55
+ allow(resp).to receive(:code).and_return(500)
56
+ allow(resp).to receive(:body).and_return("a body")
57
+
58
+ expect(subject.manticore).to receive(:get).
59
+ with(uri.to_s + "/", anything).
60
+ and_return(resp)
61
+
62
+ uri_with_path = uri.clone
63
+ uri_with_path.path = "/"
64
+
65
+ expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
66
+ with(resp.code, uri_with_path, nil, resp.body).and_call_original
67
+
68
+ expect do
69
+ subject.perform_request(uri, :get, "/")
70
+ end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
71
+ end
72
+ end
73
+
74
+ describe "format_url" do
75
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
76
+ let(:path) { "_bulk" }
77
+ subject { described_class.new(double("logger"), {}) }
78
+
79
+ it "should add the path argument to the uri's path" do
80
+ expect(subject.format_url(url, path).path).to eq("/path/_bulk")
81
+ end
82
+
83
+ context "when uri contains query parameters" do
84
+ let(:query_params) { "query=value&key=value2" }
85
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
86
+ let(:formatted) { subject.format_url(url, path)}
87
+
88
+ it "should retain query_params after format" do
89
+ expect(formatted.query).to eq(query_params)
90
+ end
91
+
92
+ context "and the path contains query parameters" do
93
+ let(:path) { "/special_path?specialParam=123" }
94
+
95
+ it "should join the query correctly" do
96
+ expect(formatted.query).to eq(query_params + "&specialParam=123")
97
+ end
98
+ end
99
+ end
100
+
101
+ context "when the path contains query parameters" do
102
+ let(:path) { "/special_bulk?pathParam=1"}
103
+ let(:formatted) { subject.format_url(url, path) }
104
+
105
+ it "should add the path correctly" do
106
+ expect(formatted.path).to eq("#{url.path}special_bulk")
107
+ expect(subject.remove_double_escaping(formatted.path)).to eq("#{url.path}special_bulk")
108
+ end
109
+
110
+ it "should add the query parameters correctly" do
111
+ expect(formatted.query).to eq("pathParam=1")
112
+ end
113
+ end
114
+
115
+ context "when uri contains credentials" do
116
+ let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
117
+ let(:formatted) { subject.format_url(url, path) }
118
+
119
+ it "should remove credentials after format" do
120
+ expect(formatted.userinfo).to be_nil
121
+ end
122
+ end
123
+
124
+ context 'when uri contains date math' do
125
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
126
+ let(:path) { CGI.escape("<logstash-{now/d}-0001>") }
127
+ let(:formatted) { subject.format_url(url, path) }
128
+
129
+ it 'should escape the uri correctly' do
130
+ expect(subject.remove_double_escaping(formatted.path)).to eq("/%3Clogstash-%7Bnow%2Fd%7D-0001%3E")
131
+ end
132
+ end
133
+
134
+ context 'when uri does not contain date math' do
135
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
136
+ let(:path) { CGI.escape("logstash-0001") }
137
+ let(:formatted) { subject.format_url(url, path) }
138
+
139
+ it 'should escape the uri correctly' do
140
+ expect(subject.remove_double_escaping(formatted.path)).to eq("/logstash-0001")
141
+ end
142
+ end
143
+ end
144
+
145
+ describe "integration specs", :integration => true do
146
+ it "should perform correct tests without error" do
147
+ resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")
148
+ expect(resp.code).to eql(200)
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,501 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/elasticsearch/http_client"
3
+ require 'cabin'
4
+
5
+ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
6
+ let(:logger) { Cabin::Channel.get }
7
+ let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
8
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
9
+ let(:options) { {:resurrect_delay => 3, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
10
+ let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'} ] }
11
+ let(:license_status) { 'active' }
12
+
13
+ subject { described_class.new(logger, adapter, initial_urls, options) }
14
+
15
+ let(:manticore_double) { double("manticore a") }
16
+ before(:each) do
17
+ response_double = double("manticore response").as_null_object
18
+ # Allow healtchecks
19
+ allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
20
+ allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
21
+ allow(manticore_double).to receive(:close)
22
+
23
+ allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
24
+
25
+ allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_version_info)
26
+ allow(subject.license_checker).to receive(:license_status).and_return(license_status)
27
+ end
28
+
29
+ after do
30
+ subject.close
31
+ end
32
+
33
+ describe "initialization" do
34
+ it "should be successful" do
35
+ expect { subject }.not_to raise_error
36
+ subject.start
37
+ end
38
+ end
39
+
40
+ describe "the resurrectionist" do
41
+ before(:each) { subject.start }
42
+ it "should start the resurrectionist when created" do
43
+ expect(subject.resurrectionist_alive?).to eql(true)
44
+ end
45
+
46
+ it "should attempt to resurrect connections after the ressurrect delay" do
47
+ expect(subject).to receive(:healthcheck!).once
48
+ sleep(subject.resurrect_delay + 1)
49
+ end
50
+
51
+ describe "healthcheck url handling" do
52
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
53
+ let(:success_response) { double("Response", :code => 200) }
54
+
55
+ before(:example) do
56
+ expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
57
+ expect(url.path).to be_empty
58
+ end
59
+ end
60
+
61
+ context "and not setting healthcheck_path" do
62
+ it "performs the healthcheck to the root" do
63
+ expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
64
+ expect(url.path).to be_empty
65
+
66
+ success_response
67
+ end
68
+ expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
69
+ end
70
+ end
71
+
72
+ context "and setting healthcheck_path" do
73
+ let(:healthcheck_path) { "/my/health" }
74
+ let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
75
+ it "performs the healthcheck to the healthcheck_path" do
76
+ expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
77
+ expect(url.path).to be_empty
78
+
79
+ success_response
80
+ end
81
+ expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
82
+ end
83
+ end
84
+ end
85
+ end
86
+
87
+ describe 'resolving the address from Elasticsearch node info' do
88
+ let(:host) { "node.elastic.co"}
89
+ let(:ip_address) { "192.168.1.0"}
90
+ let(:port) { 9200 }
91
+
92
+ context 'in Elasticsearch 1.x format' do
93
+ context 'with host and ip address' do
94
+ let(:publish_address) { "inet[#{host}/#{ip_address}:#{port}]"}
95
+ it 'should correctly extract the host' do
96
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
97
+ end
98
+ end
99
+ context 'with ip address' do
100
+ let(:publish_address) { "inet[/#{ip_address}:#{port}]"}
101
+ it 'should correctly extract the ip address' do
102
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
103
+ end
104
+ end
105
+ end
106
+
107
+ context 'in Elasticsearch 2.x-6.x format' do
108
+ let(:publish_address) { "#{ip_address}:#{port}"}
109
+ it 'should correctly extract the ip address' do
110
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("//#{ip_address}:#{port}"))
111
+ end
112
+ end
113
+
114
+ context 'in Elasticsearch 7.x'
115
+ context 'with host and ip address' do
116
+ let(:publish_address) { "#{host}/#{ip_address}:#{port}"}
117
+ it 'should correctly extract the host' do
118
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
119
+ end
120
+ end
121
+ context 'with ip address' do
122
+ let(:publish_address) { "#{ip_address}:#{port}"}
123
+ it 'should correctly extract the ip address' do
124
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
125
+ end
126
+ end
127
+ end
128
+
129
+ describe "the sniffer" do
130
+ before(:each) { subject.start }
131
+ it "should not start the sniffer by default" do
132
+ expect(subject.sniffer_alive?).to eql(nil)
133
+ end
134
+
135
+ context "when enabled" do
136
+ let(:options) { super().merge(:sniffing => true)}
137
+
138
+ it "should start the sniffer" do
139
+ expect(subject.sniffer_alive?).to eql(true)
140
+ end
141
+ end
142
+ end
143
+
144
+ describe "closing" do
145
+ before do
146
+ subject.start
147
+ # Simulate a single in use connection on the first check of this
148
+ allow(adapter).to receive(:close).and_call_original
149
+ allow(subject).to receive(:wait_for_in_use_connections).and_call_original
150
+ allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
151
+ allow(subject).to receive(:start)
152
+ subject.close
153
+ end
154
+
155
+ it "should close the adapter" do
156
+ expect(adapter).to have_received(:close)
157
+ end
158
+
159
+ it "should stop the resurrectionist" do
160
+ expect(subject.resurrectionist_alive?).to eql(false)
161
+ end
162
+
163
+ it "should stop the sniffer" do
164
+ # If no sniffer (the default) returns nil
165
+ expect(subject.sniffer_alive?).to be_falsey
166
+ end
167
+
168
+ it "should wait for in use connections to terminate" do
169
+ expect(subject).to have_received(:wait_for_in_use_connections).once
170
+ expect(subject).to have_received(:in_use_connections).twice
171
+ end
172
+ end
173
+
174
+ class MockResponse
175
+ attr_reader :code, :headers
176
+
177
+ def initialize(code = 200, body = nil, headers = {})
178
+ @code = code
179
+ @body = body
180
+ @headers = headers
181
+ end
182
+
183
+ def body
184
+ @body.to_json
185
+ end
186
+ end
187
+
188
+ describe "connection management" do
189
+ before(:each) { subject.start }
190
+ context "with only one URL in the list" do
191
+ it "should use the only URL in 'with_connection'" do
192
+ subject.with_connection do |c|
193
+ expect(c).to eq(initial_urls.first)
194
+ end
195
+ end
196
+ end
197
+
198
+ context "with multiple URLs in the list" do
199
+ let(:version_ok) do
200
+ MockResponse.new(200, {"tagline" => "You Know, for Search",
201
+ "version" => {
202
+ "number" => '7.13.0',
203
+ "build_flavor" => 'default'}
204
+ })
205
+ end
206
+ let(:success_response) { double("head_req", :code => 200)}
207
+
208
+ before :each do
209
+ allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil).and_return(success_response)
210
+ allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
211
+ end
212
+ let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
213
+
214
+ it "should minimize the number of connections to a single URL" do
215
+ connected_urls = []
216
+
217
+ # If we make 2x the number requests as we have URLs we should
218
+ # connect to each URL exactly 2 times
219
+ (initial_urls.size*2).times do
220
+ u, meta = subject.get_connection
221
+ connected_urls << u
222
+ end
223
+
224
+ connected_urls.each {|u| subject.return_connection(u) }
225
+ initial_urls.each do |url|
226
+ conn_count = connected_urls.select {|u| u == url}.size
227
+ expect(conn_count).to eql(2)
228
+ end
229
+ end
230
+
231
+ it "should correctly resurrect the dead" do
232
+ u,m = subject.get_connection
233
+
234
+ # The resurrectionist will call this to check on the backend
235
+ response = double("response", :code => 200)
236
+ expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
237
+
238
+ subject.return_connection(u)
239
+ subject.mark_dead(u, Exception.new)
240
+
241
+ expect(subject.url_meta(u)[:state]).to eql(:dead)
242
+ sleep subject.resurrect_delay + 1
243
+ expect(subject.url_meta(u)[:state]).to eql(:alive)
244
+ end
245
+ end
246
+ end
247
+
248
+ describe "version tracking" do
249
+ let(:initial_urls) { [
250
+ ::LogStash::Util::SafeURI.new("http://somehost:9200"),
251
+ ::LogStash::Util::SafeURI.new("http://otherhost:9201")
252
+ ] }
253
+
254
+ let(:valid_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
255
+ "version" => {
256
+ "number" => '7.13.0',
257
+ "build_flavor" => 'default'}
258
+ }) }
259
+
260
+ before(:each) do
261
+ allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
262
+ subject.start
263
+ end
264
+
265
+ it "picks the largest major version" do
266
+ expect(subject.maximum_seen_major_version).to eq(0)
267
+ end
268
+
269
+ context "if there are nodes with multiple major versions" do
270
+ let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'}, { "number" => '6.0.0', "build_flavor" => 'default'} ] }
271
+ it "picks the largest major version" do
272
+ expect(subject.maximum_seen_major_version).to eq(6)
273
+ end
274
+ end
275
+ end
276
+
277
+
278
+ describe "build flavour tracking" do
279
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://somehost:9200")] }
280
+
281
+ let(:es_version_info) { [ { "number" => '8.9.0', "build_flavor" => "serverless" } ] }
282
+
283
+ let(:valid_response) { MockResponse.new(200,
284
+ {"tagline" => "You Know, for Search",
285
+ "version" => {
286
+ "number" => '8.9.0',
287
+ "build_flavor" => LogStash::Outputs::ElasticSearch::HttpClient::Pool::BUILD_FLAVOUR_SERVERLESS} },
288
+ { "X-Elastic-Product" => "Elasticsearch" }
289
+ ) }
290
+
291
+ before(:each) do
292
+ allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
293
+ subject.start
294
+ end
295
+
296
+ it "picks the build flavour" do
297
+ expect(subject.serverless?).to be_truthy
298
+ end
299
+ end
300
+
301
+ describe "license checking" do
302
+ before(:each) do
303
+ allow(subject).to receive(:health_check_request)
304
+ allow(subject).to receive(:elasticsearch?).and_return(true)
305
+ end
306
+
307
+ let(:options) do
308
+ super().merge(:license_checker => license_checker)
309
+ end
310
+
311
+ context 'when LicenseChecker#acceptable_license? returns false' do
312
+ let(:license_checker) { double('LicenseChecker', :appropriate_license? => false) }
313
+
314
+ it 'does not mark the URL as active' do
315
+ subject.update_initial_urls
316
+ expect(subject.alive_urls_count).to eq(0)
317
+ end
318
+ end
319
+
320
+ context 'when LicenseChecker#acceptable_license? returns true' do
321
+ let(:license_checker) { double('LicenseChecker', :appropriate_license? => true) }
322
+
323
+ it 'marks the URL as active' do
324
+ subject.update_initial_urls
325
+ expect(subject.alive_urls_count).to eq(1)
326
+ end
327
+ end
328
+ end
329
+
330
+ # TODO: extract to ElasticSearchOutputLicenseChecker unit spec
331
+ describe "license checking with ElasticSearchOutputLicenseChecker" do
332
+ let(:options) do
333
+ super().merge(:license_checker => LogStash::Outputs::ElasticSearch::LicenseChecker.new(logger))
334
+ end
335
+
336
+ before(:each) do
337
+ allow(subject).to receive(:health_check_request)
338
+ allow(subject).to receive(:elasticsearch?).and_return(true)
339
+ end
340
+
341
+ context "if ES doesn't return a valid license" do
342
+ let(:license_status) { nil }
343
+
344
+ it "marks the url as dead" do
345
+ subject.update_initial_urls
346
+ expect(subject.alive_urls_count).to eq(0)
347
+ end
348
+
349
+ it "logs a warning" do
350
+ expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
351
+ subject.update_initial_urls
352
+ end
353
+ end
354
+
355
+ context "if ES returns a valid license" do
356
+ let(:license_status) { 'active' }
357
+
358
+ it "marks the url as active" do
359
+ subject.update_initial_urls
360
+ expect(subject.alive_urls_count).to eq(1)
361
+ end
362
+
363
+ it "does not log a warning" do
364
+ expect(subject.license_checker).to_not receive(:warn_no_license)
365
+ expect(subject.license_checker).to_not receive(:warn_invalid_license)
366
+ subject.update_initial_urls
367
+ end
368
+ end
369
+
370
+ context "if ES returns an invalid license" do
371
+ let(:license_status) { 'invalid' }
372
+
373
+ it "marks the url as active" do
374
+ subject.update_initial_urls
375
+ expect(subject.alive_urls_count).to eq(1)
376
+ end
377
+
378
+ it "logs a warning" do
379
+ expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
380
+ subject.update_initial_urls
381
+ end
382
+ end
383
+ end
384
+ end
385
+
386
+ describe "#elasticsearch?" do
387
+ let(:logger) { Cabin::Channel.get }
388
+ let(:adapter) { double("Manticore Adapter") }
389
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
390
+ let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
391
+ let(:es_version_info) { [{ "number" => '0.0.0', "build_flavor" => 'default'}] }
392
+ let(:license_status) { 'active' }
393
+
394
+ subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
395
+
396
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
397
+
398
+ context "in case HTTP error code" do
399
+ it "should fail for 401" do
400
+ allow(adapter).to receive(:perform_request)
401
+ .with(anything, :get, "/", anything, anything)
402
+ .and_return(MockResponse.new(401))
403
+
404
+ expect(subject.elasticsearch?(url)).to be false
405
+ end
406
+
407
+ it "should fail for 403" do
408
+ allow(adapter).to receive(:perform_request)
409
+ .with(anything, :get, "/", anything, anything)
410
+ .and_return(status: 403)
411
+ expect(subject.elasticsearch?(url)).to be false
412
+ end
413
+ end
414
+
415
+ context "when connecting to a cluster which reply without 'version' field" do
416
+ it "should fail" do
417
+ allow(adapter).to receive(:perform_request)
418
+ .with(anything, :get, "/", anything, anything)
419
+ .and_return(body: {"field" => "funky.com"}.to_json)
420
+ expect(subject.elasticsearch?(url)).to be false
421
+ end
422
+ end
423
+
424
+ context "when connecting to a cluster with version < 6.0.0" do
425
+ it "should fail" do
426
+ allow(adapter).to receive(:perform_request)
427
+ .with(anything, :get, "/", anything, anything)
428
+ .and_return(200, {"version" => { "number" => "5.0.0"}}.to_json)
429
+ expect(subject.elasticsearch?(url)).to be false
430
+ end
431
+ end
432
+
433
+ context "when connecting to a cluster with version in [6.0.0..7.0.0)" do
434
+ it "must be successful with valid 'tagline'" do
435
+ allow(adapter).to receive(:perform_request)
436
+ .with(anything, :get, "/", anything, anything)
437
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"}))
438
+ expect(subject.elasticsearch?(url)).to be true
439
+ end
440
+
441
+ it "should fail if invalid 'tagline'" do
442
+ allow(adapter).to receive(:perform_request)
443
+ .with(anything, :get, "/", anything, anything)
444
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"}))
445
+ expect(subject.elasticsearch?(url)).to be false
446
+ end
447
+
448
+ it "should fail if 'tagline' is not present" do
449
+ allow(adapter).to receive(:perform_request)
450
+ .with(anything, :get, "/", anything, anything)
451
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}}))
452
+ expect(subject.elasticsearch?(url)).to be false
453
+ end
454
+ end
455
+
456
+ context "when connecting to a cluster with version in [7.0.0..7.14.0)" do
457
+ it "must be successful is 'build_flavor' is 'default' and tagline is correct" do
458
+ allow(adapter).to receive(:perform_request)
459
+ .with(anything, :get, "/", anything, anything)
460
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"}))
461
+ expect(subject.elasticsearch?(url)).to be true
462
+ end
463
+
464
+ it "should fail if 'build_flavor' is not 'default' and tagline is correct" do
465
+ allow(adapter).to receive(:perform_request)
466
+ .with(anything, :get, "/", anything, anything)
467
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"}))
468
+ expect(subject.elasticsearch?(url)).to be false
469
+ end
470
+
471
+ it "should fail if 'build_flavor' is not present and tagline is correct" do
472
+ allow(adapter).to receive(:perform_request)
473
+ .with(anything, :get, "/", anything, anything)
474
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"}))
475
+ expect(subject.elasticsearch?(url)).to be false
476
+ end
477
+ end
478
+
479
+ context "when connecting to a cluster with version >= 7.14.0" do
480
+ it "should fail if 'X-elastic-product' header is not present" do
481
+ allow(adapter).to receive(:perform_request)
482
+ .with(anything, :get, "/", anything, anything)
483
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}))
484
+ expect(subject.elasticsearch?(url)).to be false
485
+ end
486
+
487
+ it "should fail if 'X-elastic-product' header is present but with bad value" do
488
+ allow(adapter).to receive(:perform_request)
489
+ .with(anything, :get, "/", anything, anything)
490
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'}))
491
+ expect(subject.elasticsearch?(url)).to be false
492
+ end
493
+
494
+ it "must be successful when 'X-elastic-product' header is present with 'Elasticsearch' value" do
495
+ allow(adapter).to receive(:perform_request)
496
+ .with(anything, :get, "/", anything, anything)
497
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'}))
498
+ expect(subject.elasticsearch?(url)).to be true
499
+ end
500
+ end
501
+ end