logstash-output-elasticsearch-test 11.16.0-x86_64-linux
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +649 -0
- data/CONTRIBUTORS +34 -0
- data/Gemfile +16 -0
- data/LICENSE +202 -0
- data/NOTICE.TXT +5 -0
- data/README.md +106 -0
- data/docs/index.asciidoc +1369 -0
- data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +282 -0
- data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +155 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +534 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +497 -0
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +201 -0
- data/lib/logstash/outputs/elasticsearch/ilm.rb +92 -0
- data/lib/logstash/outputs/elasticsearch/license_checker.rb +52 -0
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +131 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json +45 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json +44 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json +50 -0
- data/lib/logstash/outputs/elasticsearch.rb +699 -0
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +237 -0
- data/lib/logstash/plugin_mixins/elasticsearch/common.rb +409 -0
- data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb +9 -0
- data/logstash-output-elasticsearch.gemspec +40 -0
- data/spec/es_spec_helper.rb +225 -0
- data/spec/fixtures/_nodes/6x.json +81 -0
- data/spec/fixtures/_nodes/7x.json +92 -0
- data/spec/fixtures/htpasswd +2 -0
- data/spec/fixtures/license_check/active.json +16 -0
- data/spec/fixtures/license_check/inactive.json +5 -0
- data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
- data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
- data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
- data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
- data/spec/fixtures/template-with-policy-es6x.json +48 -0
- data/spec/fixtures/template-with-policy-es7x.json +45 -0
- data/spec/fixtures/template-with-policy-es8x.json +50 -0
- data/spec/fixtures/test_certs/ca.crt +29 -0
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -0
- data/spec/fixtures/test_certs/ca.key +51 -0
- data/spec/fixtures/test_certs/renew.sh +13 -0
- data/spec/fixtures/test_certs/test.crt +30 -0
- data/spec/fixtures/test_certs/test.der.sha256 +1 -0
- data/spec/fixtures/test_certs/test.key +51 -0
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/fixtures/test_certs/test_invalid.crt +36 -0
- data/spec/fixtures/test_certs/test_invalid.key +51 -0
- data/spec/fixtures/test_certs/test_invalid.p12 +0 -0
- data/spec/fixtures/test_certs/test_self_signed.crt +32 -0
- data/spec/fixtures/test_certs/test_self_signed.key +54 -0
- data/spec/fixtures/test_certs/test_self_signed.p12 +0 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +70 -0
- data/spec/integration/outputs/create_spec.rb +67 -0
- data/spec/integration/outputs/data_stream_spec.rb +68 -0
- data/spec/integration/outputs/delete_spec.rb +63 -0
- data/spec/integration/outputs/ilm_spec.rb +534 -0
- data/spec/integration/outputs/index_spec.rb +421 -0
- data/spec/integration/outputs/index_version_spec.rb +98 -0
- data/spec/integration/outputs/ingest_pipeline_spec.rb +75 -0
- data/spec/integration/outputs/metrics_spec.rb +66 -0
- data/spec/integration/outputs/no_es_on_startup_spec.rb +78 -0
- data/spec/integration/outputs/painless_update_spec.rb +99 -0
- data/spec/integration/outputs/parent_spec.rb +94 -0
- data/spec/integration/outputs/retry_spec.rb +182 -0
- data/spec/integration/outputs/routing_spec.rb +61 -0
- data/spec/integration/outputs/sniffer_spec.rb +94 -0
- data/spec/integration/outputs/templates_spec.rb +133 -0
- data/spec/integration/outputs/unsupported_actions_spec.rb +75 -0
- data/spec/integration/outputs/update_spec.rb +114 -0
- data/spec/spec_helper.rb +10 -0
- data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
- data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
- data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
- data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
- data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
- data/spec/unit/http_client_builder_spec.rb +185 -0
- data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +612 -0
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +151 -0
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +501 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +339 -0
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +189 -0
- data/spec/unit/outputs/elasticsearch_proxy_spec.rb +103 -0
- data/spec/unit/outputs/elasticsearch_spec.rb +1573 -0
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +197 -0
- data/spec/unit/outputs/error_whitelist_spec.rb +56 -0
- data/spec/unit/outputs/license_check_spec.rb +57 -0
- metadata +423 -0
@@ -0,0 +1,151 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/elasticsearch/http_client"
|
3
|
+
require 'cabin'
|
4
|
+
|
5
|
+
describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
|
6
|
+
let(:logger) { Cabin::Channel.get }
|
7
|
+
let(:options) { {} }
|
8
|
+
|
9
|
+
subject { described_class.new(logger, options) }
|
10
|
+
|
11
|
+
it "should raise an exception if requests are issued after close" do
|
12
|
+
subject.close
|
13
|
+
begin
|
14
|
+
subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/')
|
15
|
+
fail 'expected to raise a HostUnreachableError'
|
16
|
+
rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
|
17
|
+
expect( e.original_error ).to be_a ::Manticore::ClientStoppedException
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe "auth" do
|
22
|
+
let(:user) { "myuser" }
|
23
|
+
let(:password) { "mypassword" }
|
24
|
+
let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
|
25
|
+
let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
|
26
|
+
|
27
|
+
it "should convert the auth to params" do
|
28
|
+
resp = double("response")
|
29
|
+
allow(resp).to receive(:call)
|
30
|
+
allow(resp).to receive(:code).and_return(200)
|
31
|
+
|
32
|
+
expected_uri = noauth_uri.clone
|
33
|
+
expected_uri.path = "/"
|
34
|
+
|
35
|
+
expect(subject.manticore).to receive(:get).
|
36
|
+
with(expected_uri.to_s, {
|
37
|
+
:headers => {"Content-Type" => "application/json"},
|
38
|
+
:auth => {
|
39
|
+
:user => user,
|
40
|
+
:password => password,
|
41
|
+
:eager => true
|
42
|
+
}
|
43
|
+
}).and_return resp
|
44
|
+
|
45
|
+
subject.perform_request(uri, :get, "/")
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
describe "bad response codes" do
|
50
|
+
let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
51
|
+
|
52
|
+
it "should raise a bad response code error" do
|
53
|
+
resp = double("response")
|
54
|
+
allow(resp).to receive(:call)
|
55
|
+
allow(resp).to receive(:code).and_return(500)
|
56
|
+
allow(resp).to receive(:body).and_return("a body")
|
57
|
+
|
58
|
+
expect(subject.manticore).to receive(:get).
|
59
|
+
with(uri.to_s + "/", anything).
|
60
|
+
and_return(resp)
|
61
|
+
|
62
|
+
uri_with_path = uri.clone
|
63
|
+
uri_with_path.path = "/"
|
64
|
+
|
65
|
+
expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
|
66
|
+
with(resp.code, uri_with_path, nil, resp.body).and_call_original
|
67
|
+
|
68
|
+
expect do
|
69
|
+
subject.perform_request(uri, :get, "/")
|
70
|
+
end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
describe "format_url" do
|
75
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
|
76
|
+
let(:path) { "_bulk" }
|
77
|
+
subject { described_class.new(double("logger"), {}) }
|
78
|
+
|
79
|
+
it "should add the path argument to the uri's path" do
|
80
|
+
expect(subject.format_url(url, path).path).to eq("/path/_bulk")
|
81
|
+
end
|
82
|
+
|
83
|
+
context "when uri contains query parameters" do
|
84
|
+
let(:query_params) { "query=value&key=value2" }
|
85
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
|
86
|
+
let(:formatted) { subject.format_url(url, path)}
|
87
|
+
|
88
|
+
it "should retain query_params after format" do
|
89
|
+
expect(formatted.query).to eq(query_params)
|
90
|
+
end
|
91
|
+
|
92
|
+
context "and the path contains query parameters" do
|
93
|
+
let(:path) { "/special_path?specialParam=123" }
|
94
|
+
|
95
|
+
it "should join the query correctly" do
|
96
|
+
expect(formatted.query).to eq(query_params + "&specialParam=123")
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
context "when the path contains query parameters" do
|
102
|
+
let(:path) { "/special_bulk?pathParam=1"}
|
103
|
+
let(:formatted) { subject.format_url(url, path) }
|
104
|
+
|
105
|
+
it "should add the path correctly" do
|
106
|
+
expect(formatted.path).to eq("#{url.path}special_bulk")
|
107
|
+
expect(subject.remove_double_escaping(formatted.path)).to eq("#{url.path}special_bulk")
|
108
|
+
end
|
109
|
+
|
110
|
+
it "should add the query parameters correctly" do
|
111
|
+
expect(formatted.query).to eq("pathParam=1")
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
context "when uri contains credentials" do
|
116
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
|
117
|
+
let(:formatted) { subject.format_url(url, path) }
|
118
|
+
|
119
|
+
it "should remove credentials after format" do
|
120
|
+
expect(formatted.userinfo).to be_nil
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
context 'when uri contains date math' do
|
125
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
126
|
+
let(:path) { CGI.escape("<logstash-{now/d}-0001>") }
|
127
|
+
let(:formatted) { subject.format_url(url, path) }
|
128
|
+
|
129
|
+
it 'should escape the uri correctly' do
|
130
|
+
expect(subject.remove_double_escaping(formatted.path)).to eq("/%3Clogstash-%7Bnow%2Fd%7D-0001%3E")
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
context 'when uri does not contain date math' do
|
135
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
136
|
+
let(:path) { CGI.escape("logstash-0001") }
|
137
|
+
let(:formatted) { subject.format_url(url, path) }
|
138
|
+
|
139
|
+
it 'should escape the uri correctly' do
|
140
|
+
expect(subject.remove_double_escaping(formatted.path)).to eq("/logstash-0001")
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
describe "integration specs", :integration => true do
|
146
|
+
it "should perform correct tests without error" do
|
147
|
+
resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")
|
148
|
+
expect(resp.code).to eql(200)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
@@ -0,0 +1,501 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/elasticsearch/http_client"
|
3
|
+
require 'cabin'
|
4
|
+
|
5
|
+
describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
6
|
+
let(:logger) { Cabin::Channel.get }
|
7
|
+
let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
|
8
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
9
|
+
let(:options) { {:resurrect_delay => 3, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
|
10
|
+
let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'} ] }
|
11
|
+
let(:license_status) { 'active' }
|
12
|
+
|
13
|
+
subject { described_class.new(logger, adapter, initial_urls, options) }
|
14
|
+
|
15
|
+
let(:manticore_double) { double("manticore a") }
|
16
|
+
before(:each) do
|
17
|
+
response_double = double("manticore response").as_null_object
|
18
|
+
# Allow healtchecks
|
19
|
+
allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
|
20
|
+
allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
|
21
|
+
allow(manticore_double).to receive(:close)
|
22
|
+
|
23
|
+
allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
|
24
|
+
|
25
|
+
allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_version_info)
|
26
|
+
allow(subject.license_checker).to receive(:license_status).and_return(license_status)
|
27
|
+
end
|
28
|
+
|
29
|
+
after do
|
30
|
+
subject.close
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "initialization" do
|
34
|
+
it "should be successful" do
|
35
|
+
expect { subject }.not_to raise_error
|
36
|
+
subject.start
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
describe "the resurrectionist" do
|
41
|
+
before(:each) { subject.start }
|
42
|
+
it "should start the resurrectionist when created" do
|
43
|
+
expect(subject.resurrectionist_alive?).to eql(true)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should attempt to resurrect connections after the ressurrect delay" do
|
47
|
+
expect(subject).to receive(:healthcheck!).once
|
48
|
+
sleep(subject.resurrect_delay + 1)
|
49
|
+
end
|
50
|
+
|
51
|
+
describe "healthcheck url handling" do
|
52
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
53
|
+
let(:success_response) { double("Response", :code => 200) }
|
54
|
+
|
55
|
+
before(:example) do
|
56
|
+
expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
|
57
|
+
expect(url.path).to be_empty
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
context "and not setting healthcheck_path" do
|
62
|
+
it "performs the healthcheck to the root" do
|
63
|
+
expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
|
64
|
+
expect(url.path).to be_empty
|
65
|
+
|
66
|
+
success_response
|
67
|
+
end
|
68
|
+
expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context "and setting healthcheck_path" do
|
73
|
+
let(:healthcheck_path) { "/my/health" }
|
74
|
+
let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
|
75
|
+
it "performs the healthcheck to the healthcheck_path" do
|
76
|
+
expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
|
77
|
+
expect(url.path).to be_empty
|
78
|
+
|
79
|
+
success_response
|
80
|
+
end
|
81
|
+
expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
describe 'resolving the address from Elasticsearch node info' do
|
88
|
+
let(:host) { "node.elastic.co"}
|
89
|
+
let(:ip_address) { "192.168.1.0"}
|
90
|
+
let(:port) { 9200 }
|
91
|
+
|
92
|
+
context 'in Elasticsearch 1.x format' do
|
93
|
+
context 'with host and ip address' do
|
94
|
+
let(:publish_address) { "inet[#{host}/#{ip_address}:#{port}]"}
|
95
|
+
it 'should correctly extract the host' do
|
96
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
|
97
|
+
end
|
98
|
+
end
|
99
|
+
context 'with ip address' do
|
100
|
+
let(:publish_address) { "inet[/#{ip_address}:#{port}]"}
|
101
|
+
it 'should correctly extract the ip address' do
|
102
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
context 'in Elasticsearch 2.x-6.x format' do
|
108
|
+
let(:publish_address) { "#{ip_address}:#{port}"}
|
109
|
+
it 'should correctly extract the ip address' do
|
110
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("//#{ip_address}:#{port}"))
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
context 'in Elasticsearch 7.x'
|
115
|
+
context 'with host and ip address' do
|
116
|
+
let(:publish_address) { "#{host}/#{ip_address}:#{port}"}
|
117
|
+
it 'should correctly extract the host' do
|
118
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
|
119
|
+
end
|
120
|
+
end
|
121
|
+
context 'with ip address' do
|
122
|
+
let(:publish_address) { "#{ip_address}:#{port}"}
|
123
|
+
it 'should correctly extract the ip address' do
|
124
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
describe "the sniffer" do
|
130
|
+
before(:each) { subject.start }
|
131
|
+
it "should not start the sniffer by default" do
|
132
|
+
expect(subject.sniffer_alive?).to eql(nil)
|
133
|
+
end
|
134
|
+
|
135
|
+
context "when enabled" do
|
136
|
+
let(:options) { super().merge(:sniffing => true)}
|
137
|
+
|
138
|
+
it "should start the sniffer" do
|
139
|
+
expect(subject.sniffer_alive?).to eql(true)
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
describe "closing" do
|
145
|
+
before do
|
146
|
+
subject.start
|
147
|
+
# Simulate a single in use connection on the first check of this
|
148
|
+
allow(adapter).to receive(:close).and_call_original
|
149
|
+
allow(subject).to receive(:wait_for_in_use_connections).and_call_original
|
150
|
+
allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
|
151
|
+
allow(subject).to receive(:start)
|
152
|
+
subject.close
|
153
|
+
end
|
154
|
+
|
155
|
+
it "should close the adapter" do
|
156
|
+
expect(adapter).to have_received(:close)
|
157
|
+
end
|
158
|
+
|
159
|
+
it "should stop the resurrectionist" do
|
160
|
+
expect(subject.resurrectionist_alive?).to eql(false)
|
161
|
+
end
|
162
|
+
|
163
|
+
it "should stop the sniffer" do
|
164
|
+
# If no sniffer (the default) returns nil
|
165
|
+
expect(subject.sniffer_alive?).to be_falsey
|
166
|
+
end
|
167
|
+
|
168
|
+
it "should wait for in use connections to terminate" do
|
169
|
+
expect(subject).to have_received(:wait_for_in_use_connections).once
|
170
|
+
expect(subject).to have_received(:in_use_connections).twice
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
class MockResponse
|
175
|
+
attr_reader :code, :headers
|
176
|
+
|
177
|
+
def initialize(code = 200, body = nil, headers = {})
|
178
|
+
@code = code
|
179
|
+
@body = body
|
180
|
+
@headers = headers
|
181
|
+
end
|
182
|
+
|
183
|
+
def body
|
184
|
+
@body.to_json
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
describe "connection management" do
|
189
|
+
before(:each) { subject.start }
|
190
|
+
context "with only one URL in the list" do
|
191
|
+
it "should use the only URL in 'with_connection'" do
|
192
|
+
subject.with_connection do |c|
|
193
|
+
expect(c).to eq(initial_urls.first)
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
context "with multiple URLs in the list" do
|
199
|
+
let(:version_ok) do
|
200
|
+
MockResponse.new(200, {"tagline" => "You Know, for Search",
|
201
|
+
"version" => {
|
202
|
+
"number" => '7.13.0',
|
203
|
+
"build_flavor" => 'default'}
|
204
|
+
})
|
205
|
+
end
|
206
|
+
let(:success_response) { double("head_req", :code => 200)}
|
207
|
+
|
208
|
+
before :each do
|
209
|
+
allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil).and_return(success_response)
|
210
|
+
allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
|
211
|
+
end
|
212
|
+
let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
|
213
|
+
|
214
|
+
it "should minimize the number of connections to a single URL" do
|
215
|
+
connected_urls = []
|
216
|
+
|
217
|
+
# If we make 2x the number requests as we have URLs we should
|
218
|
+
# connect to each URL exactly 2 times
|
219
|
+
(initial_urls.size*2).times do
|
220
|
+
u, meta = subject.get_connection
|
221
|
+
connected_urls << u
|
222
|
+
end
|
223
|
+
|
224
|
+
connected_urls.each {|u| subject.return_connection(u) }
|
225
|
+
initial_urls.each do |url|
|
226
|
+
conn_count = connected_urls.select {|u| u == url}.size
|
227
|
+
expect(conn_count).to eql(2)
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
it "should correctly resurrect the dead" do
|
232
|
+
u,m = subject.get_connection
|
233
|
+
|
234
|
+
# The resurrectionist will call this to check on the backend
|
235
|
+
response = double("response", :code => 200)
|
236
|
+
expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
|
237
|
+
|
238
|
+
subject.return_connection(u)
|
239
|
+
subject.mark_dead(u, Exception.new)
|
240
|
+
|
241
|
+
expect(subject.url_meta(u)[:state]).to eql(:dead)
|
242
|
+
sleep subject.resurrect_delay + 1
|
243
|
+
expect(subject.url_meta(u)[:state]).to eql(:alive)
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
describe "version tracking" do
|
249
|
+
let(:initial_urls) { [
|
250
|
+
::LogStash::Util::SafeURI.new("http://somehost:9200"),
|
251
|
+
::LogStash::Util::SafeURI.new("http://otherhost:9201")
|
252
|
+
] }
|
253
|
+
|
254
|
+
let(:valid_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
|
255
|
+
"version" => {
|
256
|
+
"number" => '7.13.0',
|
257
|
+
"build_flavor" => 'default'}
|
258
|
+
}) }
|
259
|
+
|
260
|
+
before(:each) do
|
261
|
+
allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
|
262
|
+
subject.start
|
263
|
+
end
|
264
|
+
|
265
|
+
it "picks the largest major version" do
|
266
|
+
expect(subject.maximum_seen_major_version).to eq(0)
|
267
|
+
end
|
268
|
+
|
269
|
+
context "if there are nodes with multiple major versions" do
|
270
|
+
let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'}, { "number" => '6.0.0', "build_flavor" => 'default'} ] }
|
271
|
+
it "picks the largest major version" do
|
272
|
+
expect(subject.maximum_seen_major_version).to eq(6)
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
|
278
|
+
describe "build flavour tracking" do
|
279
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://somehost:9200")] }
|
280
|
+
|
281
|
+
let(:es_version_info) { [ { "number" => '8.9.0', "build_flavor" => "serverless" } ] }
|
282
|
+
|
283
|
+
let(:valid_response) { MockResponse.new(200,
|
284
|
+
{"tagline" => "You Know, for Search",
|
285
|
+
"version" => {
|
286
|
+
"number" => '8.9.0',
|
287
|
+
"build_flavor" => LogStash::Outputs::ElasticSearch::HttpClient::Pool::BUILD_FLAVOUR_SERVERLESS} },
|
288
|
+
{ "X-Elastic-Product" => "Elasticsearch" }
|
289
|
+
) }
|
290
|
+
|
291
|
+
before(:each) do
|
292
|
+
allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
|
293
|
+
subject.start
|
294
|
+
end
|
295
|
+
|
296
|
+
it "picks the build flavour" do
|
297
|
+
expect(subject.serverless?).to be_truthy
|
298
|
+
end
|
299
|
+
end
|
300
|
+
|
301
|
+
describe "license checking" do
|
302
|
+
before(:each) do
|
303
|
+
allow(subject).to receive(:health_check_request)
|
304
|
+
allow(subject).to receive(:elasticsearch?).and_return(true)
|
305
|
+
end
|
306
|
+
|
307
|
+
let(:options) do
|
308
|
+
super().merge(:license_checker => license_checker)
|
309
|
+
end
|
310
|
+
|
311
|
+
context 'when LicenseChecker#acceptable_license? returns false' do
|
312
|
+
let(:license_checker) { double('LicenseChecker', :appropriate_license? => false) }
|
313
|
+
|
314
|
+
it 'does not mark the URL as active' do
|
315
|
+
subject.update_initial_urls
|
316
|
+
expect(subject.alive_urls_count).to eq(0)
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
context 'when LicenseChecker#acceptable_license? returns true' do
|
321
|
+
let(:license_checker) { double('LicenseChecker', :appropriate_license? => true) }
|
322
|
+
|
323
|
+
it 'marks the URL as active' do
|
324
|
+
subject.update_initial_urls
|
325
|
+
expect(subject.alive_urls_count).to eq(1)
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
330
|
+
# TODO: extract to ElasticSearchOutputLicenseChecker unit spec
|
331
|
+
describe "license checking with ElasticSearchOutputLicenseChecker" do
|
332
|
+
let(:options) do
|
333
|
+
super().merge(:license_checker => LogStash::Outputs::ElasticSearch::LicenseChecker.new(logger))
|
334
|
+
end
|
335
|
+
|
336
|
+
before(:each) do
|
337
|
+
allow(subject).to receive(:health_check_request)
|
338
|
+
allow(subject).to receive(:elasticsearch?).and_return(true)
|
339
|
+
end
|
340
|
+
|
341
|
+
context "if ES doesn't return a valid license" do
|
342
|
+
let(:license_status) { nil }
|
343
|
+
|
344
|
+
it "marks the url as dead" do
|
345
|
+
subject.update_initial_urls
|
346
|
+
expect(subject.alive_urls_count).to eq(0)
|
347
|
+
end
|
348
|
+
|
349
|
+
it "logs a warning" do
|
350
|
+
expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
|
351
|
+
subject.update_initial_urls
|
352
|
+
end
|
353
|
+
end
|
354
|
+
|
355
|
+
context "if ES returns a valid license" do
|
356
|
+
let(:license_status) { 'active' }
|
357
|
+
|
358
|
+
it "marks the url as active" do
|
359
|
+
subject.update_initial_urls
|
360
|
+
expect(subject.alive_urls_count).to eq(1)
|
361
|
+
end
|
362
|
+
|
363
|
+
it "does not log a warning" do
|
364
|
+
expect(subject.license_checker).to_not receive(:warn_no_license)
|
365
|
+
expect(subject.license_checker).to_not receive(:warn_invalid_license)
|
366
|
+
subject.update_initial_urls
|
367
|
+
end
|
368
|
+
end
|
369
|
+
|
370
|
+
context "if ES returns an invalid license" do
|
371
|
+
let(:license_status) { 'invalid' }
|
372
|
+
|
373
|
+
it "marks the url as active" do
|
374
|
+
subject.update_initial_urls
|
375
|
+
expect(subject.alive_urls_count).to eq(1)
|
376
|
+
end
|
377
|
+
|
378
|
+
it "logs a warning" do
|
379
|
+
expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
|
380
|
+
subject.update_initial_urls
|
381
|
+
end
|
382
|
+
end
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
describe "#elasticsearch?" do
|
387
|
+
let(:logger) { Cabin::Channel.get }
|
388
|
+
let(:adapter) { double("Manticore Adapter") }
|
389
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
390
|
+
let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
|
391
|
+
let(:es_version_info) { [{ "number" => '0.0.0', "build_flavor" => 'default'}] }
|
392
|
+
let(:license_status) { 'active' }
|
393
|
+
|
394
|
+
subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
|
395
|
+
|
396
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
397
|
+
|
398
|
+
context "in case HTTP error code" do
|
399
|
+
it "should fail for 401" do
|
400
|
+
allow(adapter).to receive(:perform_request)
|
401
|
+
.with(anything, :get, "/", anything, anything)
|
402
|
+
.and_return(MockResponse.new(401))
|
403
|
+
|
404
|
+
expect(subject.elasticsearch?(url)).to be false
|
405
|
+
end
|
406
|
+
|
407
|
+
it "should fail for 403" do
|
408
|
+
allow(adapter).to receive(:perform_request)
|
409
|
+
.with(anything, :get, "/", anything, anything)
|
410
|
+
.and_return(status: 403)
|
411
|
+
expect(subject.elasticsearch?(url)).to be false
|
412
|
+
end
|
413
|
+
end
|
414
|
+
|
415
|
+
context "when connecting to a cluster which reply without 'version' field" do
|
416
|
+
it "should fail" do
|
417
|
+
allow(adapter).to receive(:perform_request)
|
418
|
+
.with(anything, :get, "/", anything, anything)
|
419
|
+
.and_return(body: {"field" => "funky.com"}.to_json)
|
420
|
+
expect(subject.elasticsearch?(url)).to be false
|
421
|
+
end
|
422
|
+
end
|
423
|
+
|
424
|
+
context "when connecting to a cluster with version < 6.0.0" do
|
425
|
+
it "should fail" do
|
426
|
+
allow(adapter).to receive(:perform_request)
|
427
|
+
.with(anything, :get, "/", anything, anything)
|
428
|
+
.and_return(200, {"version" => { "number" => "5.0.0"}}.to_json)
|
429
|
+
expect(subject.elasticsearch?(url)).to be false
|
430
|
+
end
|
431
|
+
end
|
432
|
+
|
433
|
+
context "when connecting to a cluster with version in [6.0.0..7.0.0)" do
|
434
|
+
it "must be successful with valid 'tagline'" do
|
435
|
+
allow(adapter).to receive(:perform_request)
|
436
|
+
.with(anything, :get, "/", anything, anything)
|
437
|
+
.and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"}))
|
438
|
+
expect(subject.elasticsearch?(url)).to be true
|
439
|
+
end
|
440
|
+
|
441
|
+
it "should fail if invalid 'tagline'" do
|
442
|
+
allow(adapter).to receive(:perform_request)
|
443
|
+
.with(anything, :get, "/", anything, anything)
|
444
|
+
.and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"}))
|
445
|
+
expect(subject.elasticsearch?(url)).to be false
|
446
|
+
end
|
447
|
+
|
448
|
+
it "should fail if 'tagline' is not present" do
|
449
|
+
allow(adapter).to receive(:perform_request)
|
450
|
+
.with(anything, :get, "/", anything, anything)
|
451
|
+
.and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}}))
|
452
|
+
expect(subject.elasticsearch?(url)).to be false
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
context "when connecting to a cluster with version in [7.0.0..7.14.0)" do
|
457
|
+
it "must be successful is 'build_flavor' is 'default' and tagline is correct" do
|
458
|
+
allow(adapter).to receive(:perform_request)
|
459
|
+
.with(anything, :get, "/", anything, anything)
|
460
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"}))
|
461
|
+
expect(subject.elasticsearch?(url)).to be true
|
462
|
+
end
|
463
|
+
|
464
|
+
it "should fail if 'build_flavor' is not 'default' and tagline is correct" do
|
465
|
+
allow(adapter).to receive(:perform_request)
|
466
|
+
.with(anything, :get, "/", anything, anything)
|
467
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"}))
|
468
|
+
expect(subject.elasticsearch?(url)).to be false
|
469
|
+
end
|
470
|
+
|
471
|
+
it "should fail if 'build_flavor' is not present and tagline is correct" do
|
472
|
+
allow(adapter).to receive(:perform_request)
|
473
|
+
.with(anything, :get, "/", anything, anything)
|
474
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"}))
|
475
|
+
expect(subject.elasticsearch?(url)).to be false
|
476
|
+
end
|
477
|
+
end
|
478
|
+
|
479
|
+
context "when connecting to a cluster with version >= 7.14.0" do
|
480
|
+
it "should fail if 'X-elastic-product' header is not present" do
|
481
|
+
allow(adapter).to receive(:perform_request)
|
482
|
+
.with(anything, :get, "/", anything, anything)
|
483
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}))
|
484
|
+
expect(subject.elasticsearch?(url)).to be false
|
485
|
+
end
|
486
|
+
|
487
|
+
it "should fail if 'X-elastic-product' header is present but with bad value" do
|
488
|
+
allow(adapter).to receive(:perform_request)
|
489
|
+
.with(anything, :get, "/", anything, anything)
|
490
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'}))
|
491
|
+
expect(subject.elasticsearch?(url)).to be false
|
492
|
+
end
|
493
|
+
|
494
|
+
it "must be successful when 'X-elastic-product' header is present with 'Elasticsearch' value" do
|
495
|
+
allow(adapter).to receive(:perform_request)
|
496
|
+
.with(anything, :get, "/", anything, anything)
|
497
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'}))
|
498
|
+
expect(subject.elasticsearch?(url)).to be true
|
499
|
+
end
|
500
|
+
end
|
501
|
+
end
|