logstash-output-elasticsearch-test 11.16.0-x86_64-linux

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +649 -0
  3. data/CONTRIBUTORS +34 -0
  4. data/Gemfile +16 -0
  5. data/LICENSE +202 -0
  6. data/NOTICE.TXT +5 -0
  7. data/README.md +106 -0
  8. data/docs/index.asciidoc +1369 -0
  9. data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +282 -0
  10. data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
  11. data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +155 -0
  12. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +534 -0
  13. data/lib/logstash/outputs/elasticsearch/http_client.rb +497 -0
  14. data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +201 -0
  15. data/lib/logstash/outputs/elasticsearch/ilm.rb +92 -0
  16. data/lib/logstash/outputs/elasticsearch/license_checker.rb +52 -0
  17. data/lib/logstash/outputs/elasticsearch/template_manager.rb +131 -0
  18. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json +45 -0
  19. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json +44 -0
  20. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json +50 -0
  21. data/lib/logstash/outputs/elasticsearch.rb +699 -0
  22. data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +237 -0
  23. data/lib/logstash/plugin_mixins/elasticsearch/common.rb +409 -0
  24. data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb +9 -0
  25. data/logstash-output-elasticsearch.gemspec +40 -0
  26. data/spec/es_spec_helper.rb +225 -0
  27. data/spec/fixtures/_nodes/6x.json +81 -0
  28. data/spec/fixtures/_nodes/7x.json +92 -0
  29. data/spec/fixtures/htpasswd +2 -0
  30. data/spec/fixtures/license_check/active.json +16 -0
  31. data/spec/fixtures/license_check/inactive.json +5 -0
  32. data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
  33. data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
  34. data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
  35. data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
  36. data/spec/fixtures/template-with-policy-es6x.json +48 -0
  37. data/spec/fixtures/template-with-policy-es7x.json +45 -0
  38. data/spec/fixtures/template-with-policy-es8x.json +50 -0
  39. data/spec/fixtures/test_certs/ca.crt +29 -0
  40. data/spec/fixtures/test_certs/ca.der.sha256 +1 -0
  41. data/spec/fixtures/test_certs/ca.key +51 -0
  42. data/spec/fixtures/test_certs/renew.sh +13 -0
  43. data/spec/fixtures/test_certs/test.crt +30 -0
  44. data/spec/fixtures/test_certs/test.der.sha256 +1 -0
  45. data/spec/fixtures/test_certs/test.key +51 -0
  46. data/spec/fixtures/test_certs/test.p12 +0 -0
  47. data/spec/fixtures/test_certs/test_invalid.crt +36 -0
  48. data/spec/fixtures/test_certs/test_invalid.key +51 -0
  49. data/spec/fixtures/test_certs/test_invalid.p12 +0 -0
  50. data/spec/fixtures/test_certs/test_self_signed.crt +32 -0
  51. data/spec/fixtures/test_certs/test_self_signed.key +54 -0
  52. data/spec/fixtures/test_certs/test_self_signed.p12 +0 -0
  53. data/spec/integration/outputs/compressed_indexing_spec.rb +70 -0
  54. data/spec/integration/outputs/create_spec.rb +67 -0
  55. data/spec/integration/outputs/data_stream_spec.rb +68 -0
  56. data/spec/integration/outputs/delete_spec.rb +63 -0
  57. data/spec/integration/outputs/ilm_spec.rb +534 -0
  58. data/spec/integration/outputs/index_spec.rb +421 -0
  59. data/spec/integration/outputs/index_version_spec.rb +98 -0
  60. data/spec/integration/outputs/ingest_pipeline_spec.rb +75 -0
  61. data/spec/integration/outputs/metrics_spec.rb +66 -0
  62. data/spec/integration/outputs/no_es_on_startup_spec.rb +78 -0
  63. data/spec/integration/outputs/painless_update_spec.rb +99 -0
  64. data/spec/integration/outputs/parent_spec.rb +94 -0
  65. data/spec/integration/outputs/retry_spec.rb +182 -0
  66. data/spec/integration/outputs/routing_spec.rb +61 -0
  67. data/spec/integration/outputs/sniffer_spec.rb +94 -0
  68. data/spec/integration/outputs/templates_spec.rb +133 -0
  69. data/spec/integration/outputs/unsupported_actions_spec.rb +75 -0
  70. data/spec/integration/outputs/update_spec.rb +114 -0
  71. data/spec/spec_helper.rb +10 -0
  72. data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
  73. data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
  74. data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
  75. data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
  76. data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
  77. data/spec/unit/http_client_builder_spec.rb +185 -0
  78. data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +612 -0
  79. data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +151 -0
  80. data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +501 -0
  81. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +339 -0
  82. data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +189 -0
  83. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +103 -0
  84. data/spec/unit/outputs/elasticsearch_spec.rb +1573 -0
  85. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +197 -0
  86. data/spec/unit/outputs/error_whitelist_spec.rb +56 -0
  87. data/spec/unit/outputs/license_check_spec.rb +57 -0
  88. metadata +423 -0
@@ -0,0 +1,151 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/elasticsearch/http_client"
3
+ require 'cabin'
4
+
5
+ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
6
+ let(:logger) { Cabin::Channel.get }
7
+ let(:options) { {} }
8
+
9
+ subject { described_class.new(logger, options) }
10
+
11
+ it "should raise an exception if requests are issued after close" do
12
+ subject.close
13
+ begin
14
+ subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/')
15
+ fail 'expected to raise a HostUnreachableError'
16
+ rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
17
+ expect( e.original_error ).to be_a ::Manticore::ClientStoppedException
18
+ end
19
+ end
20
+
21
+ describe "auth" do
22
+ let(:user) { "myuser" }
23
+ let(:password) { "mypassword" }
24
+ let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
25
+ let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
26
+
27
+ it "should convert the auth to params" do
28
+ resp = double("response")
29
+ allow(resp).to receive(:call)
30
+ allow(resp).to receive(:code).and_return(200)
31
+
32
+ expected_uri = noauth_uri.clone
33
+ expected_uri.path = "/"
34
+
35
+ expect(subject.manticore).to receive(:get).
36
+ with(expected_uri.to_s, {
37
+ :headers => {"Content-Type" => "application/json"},
38
+ :auth => {
39
+ :user => user,
40
+ :password => password,
41
+ :eager => true
42
+ }
43
+ }).and_return resp
44
+
45
+ subject.perform_request(uri, :get, "/")
46
+ end
47
+ end
48
+
49
+ describe "bad response codes" do
50
+ let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
51
+
52
+ it "should raise a bad response code error" do
53
+ resp = double("response")
54
+ allow(resp).to receive(:call)
55
+ allow(resp).to receive(:code).and_return(500)
56
+ allow(resp).to receive(:body).and_return("a body")
57
+
58
+ expect(subject.manticore).to receive(:get).
59
+ with(uri.to_s + "/", anything).
60
+ and_return(resp)
61
+
62
+ uri_with_path = uri.clone
63
+ uri_with_path.path = "/"
64
+
65
+ expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
66
+ with(resp.code, uri_with_path, nil, resp.body).and_call_original
67
+
68
+ expect do
69
+ subject.perform_request(uri, :get, "/")
70
+ end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
71
+ end
72
+ end
73
+
74
+ describe "format_url" do
75
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
76
+ let(:path) { "_bulk" }
77
+ subject { described_class.new(double("logger"), {}) }
78
+
79
+ it "should add the path argument to the uri's path" do
80
+ expect(subject.format_url(url, path).path).to eq("/path/_bulk")
81
+ end
82
+
83
+ context "when uri contains query parameters" do
84
+ let(:query_params) { "query=value&key=value2" }
85
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
86
+ let(:formatted) { subject.format_url(url, path)}
87
+
88
+ it "should retain query_params after format" do
89
+ expect(formatted.query).to eq(query_params)
90
+ end
91
+
92
+ context "and the path contains query parameters" do
93
+ let(:path) { "/special_path?specialParam=123" }
94
+
95
+ it "should join the query correctly" do
96
+ expect(formatted.query).to eq(query_params + "&specialParam=123")
97
+ end
98
+ end
99
+ end
100
+
101
+ context "when the path contains query parameters" do
102
+ let(:path) { "/special_bulk?pathParam=1"}
103
+ let(:formatted) { subject.format_url(url, path) }
104
+
105
+ it "should add the path correctly" do
106
+ expect(formatted.path).to eq("#{url.path}special_bulk")
107
+ expect(subject.remove_double_escaping(formatted.path)).to eq("#{url.path}special_bulk")
108
+ end
109
+
110
+ it "should add the query parameters correctly" do
111
+ expect(formatted.query).to eq("pathParam=1")
112
+ end
113
+ end
114
+
115
+ context "when uri contains credentials" do
116
+ let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
117
+ let(:formatted) { subject.format_url(url, path) }
118
+
119
+ it "should remove credentials after format" do
120
+ expect(formatted.userinfo).to be_nil
121
+ end
122
+ end
123
+
124
+ context 'when uri contains date math' do
125
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
126
+ let(:path) { CGI.escape("<logstash-{now/d}-0001>") }
127
+ let(:formatted) { subject.format_url(url, path) }
128
+
129
+ it 'should escape the uri correctly' do
130
+ expect(subject.remove_double_escaping(formatted.path)).to eq("/%3Clogstash-%7Bnow%2Fd%7D-0001%3E")
131
+ end
132
+ end
133
+
134
+ context 'when uri does not contain date math' do
135
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
136
+ let(:path) { CGI.escape("logstash-0001") }
137
+ let(:formatted) { subject.format_url(url, path) }
138
+
139
+ it 'should escape the uri correctly' do
140
+ expect(subject.remove_double_escaping(formatted.path)).to eq("/logstash-0001")
141
+ end
142
+ end
143
+ end
144
+
145
+ describe "integration specs", :integration => true do
146
+ it "should perform correct tests without error" do
147
+ resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")
148
+ expect(resp.code).to eql(200)
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,501 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/elasticsearch/http_client"
3
+ require 'cabin'
4
+
5
+ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
6
+ let(:logger) { Cabin::Channel.get }
7
+ let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
8
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
9
+ let(:options) { {:resurrect_delay => 3, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
10
+ let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'} ] }
11
+ let(:license_status) { 'active' }
12
+
13
+ subject { described_class.new(logger, adapter, initial_urls, options) }
14
+
15
+ let(:manticore_double) { double("manticore a") }
16
+ before(:each) do
17
+ response_double = double("manticore response").as_null_object
18
+ # Allow healtchecks
19
+ allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
20
+ allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
21
+ allow(manticore_double).to receive(:close)
22
+
23
+ allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
24
+
25
+ allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_version_info)
26
+ allow(subject.license_checker).to receive(:license_status).and_return(license_status)
27
+ end
28
+
29
+ after do
30
+ subject.close
31
+ end
32
+
33
+ describe "initialization" do
34
+ it "should be successful" do
35
+ expect { subject }.not_to raise_error
36
+ subject.start
37
+ end
38
+ end
39
+
40
+ describe "the resurrectionist" do
41
+ before(:each) { subject.start }
42
+ it "should start the resurrectionist when created" do
43
+ expect(subject.resurrectionist_alive?).to eql(true)
44
+ end
45
+
46
+ it "should attempt to resurrect connections after the ressurrect delay" do
47
+ expect(subject).to receive(:healthcheck!).once
48
+ sleep(subject.resurrect_delay + 1)
49
+ end
50
+
51
+ describe "healthcheck url handling" do
52
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
53
+ let(:success_response) { double("Response", :code => 200) }
54
+
55
+ before(:example) do
56
+ expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
57
+ expect(url.path).to be_empty
58
+ end
59
+ end
60
+
61
+ context "and not setting healthcheck_path" do
62
+ it "performs the healthcheck to the root" do
63
+ expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
64
+ expect(url.path).to be_empty
65
+
66
+ success_response
67
+ end
68
+ expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
69
+ end
70
+ end
71
+
72
+ context "and setting healthcheck_path" do
73
+ let(:healthcheck_path) { "/my/health" }
74
+ let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
75
+ it "performs the healthcheck to the healthcheck_path" do
76
+ expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
77
+ expect(url.path).to be_empty
78
+
79
+ success_response
80
+ end
81
+ expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
82
+ end
83
+ end
84
+ end
85
+ end
86
+
87
+ describe 'resolving the address from Elasticsearch node info' do
88
+ let(:host) { "node.elastic.co"}
89
+ let(:ip_address) { "192.168.1.0"}
90
+ let(:port) { 9200 }
91
+
92
+ context 'in Elasticsearch 1.x format' do
93
+ context 'with host and ip address' do
94
+ let(:publish_address) { "inet[#{host}/#{ip_address}:#{port}]"}
95
+ it 'should correctly extract the host' do
96
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
97
+ end
98
+ end
99
+ context 'with ip address' do
100
+ let(:publish_address) { "inet[/#{ip_address}:#{port}]"}
101
+ it 'should correctly extract the ip address' do
102
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
103
+ end
104
+ end
105
+ end
106
+
107
+ context 'in Elasticsearch 2.x-6.x format' do
108
+ let(:publish_address) { "#{ip_address}:#{port}"}
109
+ it 'should correctly extract the ip address' do
110
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("//#{ip_address}:#{port}"))
111
+ end
112
+ end
113
+
114
+ context 'in Elasticsearch 7.x'
115
+ context 'with host and ip address' do
116
+ let(:publish_address) { "#{host}/#{ip_address}:#{port}"}
117
+ it 'should correctly extract the host' do
118
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
119
+ end
120
+ end
121
+ context 'with ip address' do
122
+ let(:publish_address) { "#{ip_address}:#{port}"}
123
+ it 'should correctly extract the ip address' do
124
+ expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
125
+ end
126
+ end
127
+ end
128
+
129
+ describe "the sniffer" do
130
+ before(:each) { subject.start }
131
+ it "should not start the sniffer by default" do
132
+ expect(subject.sniffer_alive?).to eql(nil)
133
+ end
134
+
135
+ context "when enabled" do
136
+ let(:options) { super().merge(:sniffing => true)}
137
+
138
+ it "should start the sniffer" do
139
+ expect(subject.sniffer_alive?).to eql(true)
140
+ end
141
+ end
142
+ end
143
+
144
+ describe "closing" do
145
+ before do
146
+ subject.start
147
+ # Simulate a single in use connection on the first check of this
148
+ allow(adapter).to receive(:close).and_call_original
149
+ allow(subject).to receive(:wait_for_in_use_connections).and_call_original
150
+ allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
151
+ allow(subject).to receive(:start)
152
+ subject.close
153
+ end
154
+
155
+ it "should close the adapter" do
156
+ expect(adapter).to have_received(:close)
157
+ end
158
+
159
+ it "should stop the resurrectionist" do
160
+ expect(subject.resurrectionist_alive?).to eql(false)
161
+ end
162
+
163
+ it "should stop the sniffer" do
164
+ # If no sniffer (the default) returns nil
165
+ expect(subject.sniffer_alive?).to be_falsey
166
+ end
167
+
168
+ it "should wait for in use connections to terminate" do
169
+ expect(subject).to have_received(:wait_for_in_use_connections).once
170
+ expect(subject).to have_received(:in_use_connections).twice
171
+ end
172
+ end
173
+
174
+ class MockResponse
175
+ attr_reader :code, :headers
176
+
177
+ def initialize(code = 200, body = nil, headers = {})
178
+ @code = code
179
+ @body = body
180
+ @headers = headers
181
+ end
182
+
183
+ def body
184
+ @body.to_json
185
+ end
186
+ end
187
+
188
+ describe "connection management" do
189
+ before(:each) { subject.start }
190
+ context "with only one URL in the list" do
191
+ it "should use the only URL in 'with_connection'" do
192
+ subject.with_connection do |c|
193
+ expect(c).to eq(initial_urls.first)
194
+ end
195
+ end
196
+ end
197
+
198
+ context "with multiple URLs in the list" do
199
+ let(:version_ok) do
200
+ MockResponse.new(200, {"tagline" => "You Know, for Search",
201
+ "version" => {
202
+ "number" => '7.13.0',
203
+ "build_flavor" => 'default'}
204
+ })
205
+ end
206
+ let(:success_response) { double("head_req", :code => 200)}
207
+
208
+ before :each do
209
+ allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil).and_return(success_response)
210
+ allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
211
+ end
212
+ let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
213
+
214
+ it "should minimize the number of connections to a single URL" do
215
+ connected_urls = []
216
+
217
+ # If we make 2x the number requests as we have URLs we should
218
+ # connect to each URL exactly 2 times
219
+ (initial_urls.size*2).times do
220
+ u, meta = subject.get_connection
221
+ connected_urls << u
222
+ end
223
+
224
+ connected_urls.each {|u| subject.return_connection(u) }
225
+ initial_urls.each do |url|
226
+ conn_count = connected_urls.select {|u| u == url}.size
227
+ expect(conn_count).to eql(2)
228
+ end
229
+ end
230
+
231
+ it "should correctly resurrect the dead" do
232
+ u,m = subject.get_connection
233
+
234
+ # The resurrectionist will call this to check on the backend
235
+ response = double("response", :code => 200)
236
+ expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
237
+
238
+ subject.return_connection(u)
239
+ subject.mark_dead(u, Exception.new)
240
+
241
+ expect(subject.url_meta(u)[:state]).to eql(:dead)
242
+ sleep subject.resurrect_delay + 1
243
+ expect(subject.url_meta(u)[:state]).to eql(:alive)
244
+ end
245
+ end
246
+ end
247
+
248
+ describe "version tracking" do
249
+ let(:initial_urls) { [
250
+ ::LogStash::Util::SafeURI.new("http://somehost:9200"),
251
+ ::LogStash::Util::SafeURI.new("http://otherhost:9201")
252
+ ] }
253
+
254
+ let(:valid_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
255
+ "version" => {
256
+ "number" => '7.13.0',
257
+ "build_flavor" => 'default'}
258
+ }) }
259
+
260
+ before(:each) do
261
+ allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
262
+ subject.start
263
+ end
264
+
265
+ it "picks the largest major version" do
266
+ expect(subject.maximum_seen_major_version).to eq(0)
267
+ end
268
+
269
+ context "if there are nodes with multiple major versions" do
270
+ let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'}, { "number" => '6.0.0', "build_flavor" => 'default'} ] }
271
+ it "picks the largest major version" do
272
+ expect(subject.maximum_seen_major_version).to eq(6)
273
+ end
274
+ end
275
+ end
276
+
277
+
278
+ describe "build flavour tracking" do
279
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://somehost:9200")] }
280
+
281
+ let(:es_version_info) { [ { "number" => '8.9.0', "build_flavor" => "serverless" } ] }
282
+
283
+ let(:valid_response) { MockResponse.new(200,
284
+ {"tagline" => "You Know, for Search",
285
+ "version" => {
286
+ "number" => '8.9.0',
287
+ "build_flavor" => LogStash::Outputs::ElasticSearch::HttpClient::Pool::BUILD_FLAVOUR_SERVERLESS} },
288
+ { "X-Elastic-Product" => "Elasticsearch" }
289
+ ) }
290
+
291
+ before(:each) do
292
+ allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
293
+ subject.start
294
+ end
295
+
296
+ it "picks the build flavour" do
297
+ expect(subject.serverless?).to be_truthy
298
+ end
299
+ end
300
+
301
+ describe "license checking" do
302
+ before(:each) do
303
+ allow(subject).to receive(:health_check_request)
304
+ allow(subject).to receive(:elasticsearch?).and_return(true)
305
+ end
306
+
307
+ let(:options) do
308
+ super().merge(:license_checker => license_checker)
309
+ end
310
+
311
+ context 'when LicenseChecker#acceptable_license? returns false' do
312
+ let(:license_checker) { double('LicenseChecker', :appropriate_license? => false) }
313
+
314
+ it 'does not mark the URL as active' do
315
+ subject.update_initial_urls
316
+ expect(subject.alive_urls_count).to eq(0)
317
+ end
318
+ end
319
+
320
+ context 'when LicenseChecker#acceptable_license? returns true' do
321
+ let(:license_checker) { double('LicenseChecker', :appropriate_license? => true) }
322
+
323
+ it 'marks the URL as active' do
324
+ subject.update_initial_urls
325
+ expect(subject.alive_urls_count).to eq(1)
326
+ end
327
+ end
328
+ end
329
+
330
+ # TODO: extract to ElasticSearchOutputLicenseChecker unit spec
331
+ describe "license checking with ElasticSearchOutputLicenseChecker" do
332
+ let(:options) do
333
+ super().merge(:license_checker => LogStash::Outputs::ElasticSearch::LicenseChecker.new(logger))
334
+ end
335
+
336
+ before(:each) do
337
+ allow(subject).to receive(:health_check_request)
338
+ allow(subject).to receive(:elasticsearch?).and_return(true)
339
+ end
340
+
341
+ context "if ES doesn't return a valid license" do
342
+ let(:license_status) { nil }
343
+
344
+ it "marks the url as dead" do
345
+ subject.update_initial_urls
346
+ expect(subject.alive_urls_count).to eq(0)
347
+ end
348
+
349
+ it "logs a warning" do
350
+ expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
351
+ subject.update_initial_urls
352
+ end
353
+ end
354
+
355
+ context "if ES returns a valid license" do
356
+ let(:license_status) { 'active' }
357
+
358
+ it "marks the url as active" do
359
+ subject.update_initial_urls
360
+ expect(subject.alive_urls_count).to eq(1)
361
+ end
362
+
363
+ it "does not log a warning" do
364
+ expect(subject.license_checker).to_not receive(:warn_no_license)
365
+ expect(subject.license_checker).to_not receive(:warn_invalid_license)
366
+ subject.update_initial_urls
367
+ end
368
+ end
369
+
370
+ context "if ES returns an invalid license" do
371
+ let(:license_status) { 'invalid' }
372
+
373
+ it "marks the url as active" do
374
+ subject.update_initial_urls
375
+ expect(subject.alive_urls_count).to eq(1)
376
+ end
377
+
378
+ it "logs a warning" do
379
+ expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
380
+ subject.update_initial_urls
381
+ end
382
+ end
383
+ end
384
+ end
385
+
386
+ describe "#elasticsearch?" do
387
+ let(:logger) { Cabin::Channel.get }
388
+ let(:adapter) { double("Manticore Adapter") }
389
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
390
+ let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
391
+ let(:es_version_info) { [{ "number" => '0.0.0', "build_flavor" => 'default'}] }
392
+ let(:license_status) { 'active' }
393
+
394
+ subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
395
+
396
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
397
+
398
+ context "in case HTTP error code" do
399
+ it "should fail for 401" do
400
+ allow(adapter).to receive(:perform_request)
401
+ .with(anything, :get, "/", anything, anything)
402
+ .and_return(MockResponse.new(401))
403
+
404
+ expect(subject.elasticsearch?(url)).to be false
405
+ end
406
+
407
+ it "should fail for 403" do
408
+ allow(adapter).to receive(:perform_request)
409
+ .with(anything, :get, "/", anything, anything)
410
+ .and_return(status: 403)
411
+ expect(subject.elasticsearch?(url)).to be false
412
+ end
413
+ end
414
+
415
+ context "when connecting to a cluster which reply without 'version' field" do
416
+ it "should fail" do
417
+ allow(adapter).to receive(:perform_request)
418
+ .with(anything, :get, "/", anything, anything)
419
+ .and_return(body: {"field" => "funky.com"}.to_json)
420
+ expect(subject.elasticsearch?(url)).to be false
421
+ end
422
+ end
423
+
424
+ context "when connecting to a cluster with version < 6.0.0" do
425
+ it "should fail" do
426
+ allow(adapter).to receive(:perform_request)
427
+ .with(anything, :get, "/", anything, anything)
428
+ .and_return(200, {"version" => { "number" => "5.0.0"}}.to_json)
429
+ expect(subject.elasticsearch?(url)).to be false
430
+ end
431
+ end
432
+
433
+ context "when connecting to a cluster with version in [6.0.0..7.0.0)" do
434
+ it "must be successful with valid 'tagline'" do
435
+ allow(adapter).to receive(:perform_request)
436
+ .with(anything, :get, "/", anything, anything)
437
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"}))
438
+ expect(subject.elasticsearch?(url)).to be true
439
+ end
440
+
441
+ it "should fail if invalid 'tagline'" do
442
+ allow(adapter).to receive(:perform_request)
443
+ .with(anything, :get, "/", anything, anything)
444
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"}))
445
+ expect(subject.elasticsearch?(url)).to be false
446
+ end
447
+
448
+ it "should fail if 'tagline' is not present" do
449
+ allow(adapter).to receive(:perform_request)
450
+ .with(anything, :get, "/", anything, anything)
451
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}}))
452
+ expect(subject.elasticsearch?(url)).to be false
453
+ end
454
+ end
455
+
456
+ context "when connecting to a cluster with version in [7.0.0..7.14.0)" do
457
+ it "must be successful is 'build_flavor' is 'default' and tagline is correct" do
458
+ allow(adapter).to receive(:perform_request)
459
+ .with(anything, :get, "/", anything, anything)
460
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"}))
461
+ expect(subject.elasticsearch?(url)).to be true
462
+ end
463
+
464
+ it "should fail if 'build_flavor' is not 'default' and tagline is correct" do
465
+ allow(adapter).to receive(:perform_request)
466
+ .with(anything, :get, "/", anything, anything)
467
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"}))
468
+ expect(subject.elasticsearch?(url)).to be false
469
+ end
470
+
471
+ it "should fail if 'build_flavor' is not present and tagline is correct" do
472
+ allow(adapter).to receive(:perform_request)
473
+ .with(anything, :get, "/", anything, anything)
474
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"}))
475
+ expect(subject.elasticsearch?(url)).to be false
476
+ end
477
+ end
478
+
479
+ context "when connecting to a cluster with version >= 7.14.0" do
480
+ it "should fail if 'X-elastic-product' header is not present" do
481
+ allow(adapter).to receive(:perform_request)
482
+ .with(anything, :get, "/", anything, anything)
483
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}))
484
+ expect(subject.elasticsearch?(url)).to be false
485
+ end
486
+
487
+ it "should fail if 'X-elastic-product' header is present but with bad value" do
488
+ allow(adapter).to receive(:perform_request)
489
+ .with(anything, :get, "/", anything, anything)
490
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'}))
491
+ expect(subject.elasticsearch?(url)).to be false
492
+ end
493
+
494
+ it "must be successful when 'X-elastic-product' header is present with 'Elasticsearch' value" do
495
+ allow(adapter).to receive(:perform_request)
496
+ .with(anything, :get, "/", anything, anything)
497
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'}))
498
+ expect(subject.elasticsearch?(url)).to be true
499
+ end
500
+ end
501
+ end