logstash-output-elasticsearch-test 11.16.0-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +649 -0
- data/CONTRIBUTORS +34 -0
- data/Gemfile +16 -0
- data/LICENSE +202 -0
- data/NOTICE.TXT +5 -0
- data/README.md +106 -0
- data/docs/index.asciidoc +1369 -0
- data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +282 -0
- data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +155 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +534 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +497 -0
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +201 -0
- data/lib/logstash/outputs/elasticsearch/ilm.rb +92 -0
- data/lib/logstash/outputs/elasticsearch/license_checker.rb +52 -0
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +131 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json +45 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json +44 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json +50 -0
- data/lib/logstash/outputs/elasticsearch.rb +699 -0
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +237 -0
- data/lib/logstash/plugin_mixins/elasticsearch/common.rb +409 -0
- data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb +9 -0
- data/logstash-output-elasticsearch.gemspec +40 -0
- data/spec/es_spec_helper.rb +225 -0
- data/spec/fixtures/_nodes/6x.json +81 -0
- data/spec/fixtures/_nodes/7x.json +92 -0
- data/spec/fixtures/htpasswd +2 -0
- data/spec/fixtures/license_check/active.json +16 -0
- data/spec/fixtures/license_check/inactive.json +5 -0
- data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
- data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
- data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
- data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
- data/spec/fixtures/template-with-policy-es6x.json +48 -0
- data/spec/fixtures/template-with-policy-es7x.json +45 -0
- data/spec/fixtures/template-with-policy-es8x.json +50 -0
- data/spec/fixtures/test_certs/ca.crt +29 -0
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -0
- data/spec/fixtures/test_certs/ca.key +51 -0
- data/spec/fixtures/test_certs/renew.sh +13 -0
- data/spec/fixtures/test_certs/test.crt +30 -0
- data/spec/fixtures/test_certs/test.der.sha256 +1 -0
- data/spec/fixtures/test_certs/test.key +51 -0
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/fixtures/test_certs/test_invalid.crt +36 -0
- data/spec/fixtures/test_certs/test_invalid.key +51 -0
- data/spec/fixtures/test_certs/test_invalid.p12 +0 -0
- data/spec/fixtures/test_certs/test_self_signed.crt +32 -0
- data/spec/fixtures/test_certs/test_self_signed.key +54 -0
- data/spec/fixtures/test_certs/test_self_signed.p12 +0 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +70 -0
- data/spec/integration/outputs/create_spec.rb +67 -0
- data/spec/integration/outputs/data_stream_spec.rb +68 -0
- data/spec/integration/outputs/delete_spec.rb +63 -0
- data/spec/integration/outputs/ilm_spec.rb +534 -0
- data/spec/integration/outputs/index_spec.rb +421 -0
- data/spec/integration/outputs/index_version_spec.rb +98 -0
- data/spec/integration/outputs/ingest_pipeline_spec.rb +75 -0
- data/spec/integration/outputs/metrics_spec.rb +66 -0
- data/spec/integration/outputs/no_es_on_startup_spec.rb +78 -0
- data/spec/integration/outputs/painless_update_spec.rb +99 -0
- data/spec/integration/outputs/parent_spec.rb +94 -0
- data/spec/integration/outputs/retry_spec.rb +182 -0
- data/spec/integration/outputs/routing_spec.rb +61 -0
- data/spec/integration/outputs/sniffer_spec.rb +94 -0
- data/spec/integration/outputs/templates_spec.rb +133 -0
- data/spec/integration/outputs/unsupported_actions_spec.rb +75 -0
- data/spec/integration/outputs/update_spec.rb +114 -0
- data/spec/spec_helper.rb +10 -0
- data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
- data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
- data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
- data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
- data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
- data/spec/unit/http_client_builder_spec.rb +185 -0
- data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +612 -0
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +151 -0
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +501 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +339 -0
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +189 -0
- data/spec/unit/outputs/elasticsearch_proxy_spec.rb +103 -0
- data/spec/unit/outputs/elasticsearch_spec.rb +1573 -0
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +197 -0
- data/spec/unit/outputs/error_whitelist_spec.rb +56 -0
- data/spec/unit/outputs/license_check_spec.rb +57 -0
- metadata +423 -0
@@ -0,0 +1,151 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/elasticsearch/http_client"
|
3
|
+
require 'cabin'
|
4
|
+
|
5
|
+
describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
|
6
|
+
let(:logger) { Cabin::Channel.get }
|
7
|
+
let(:options) { {} }
|
8
|
+
|
9
|
+
subject { described_class.new(logger, options) }
|
10
|
+
|
11
|
+
it "should raise an exception if requests are issued after close" do
|
12
|
+
subject.close
|
13
|
+
begin
|
14
|
+
subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/')
|
15
|
+
fail 'expected to raise a HostUnreachableError'
|
16
|
+
rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
|
17
|
+
expect( e.original_error ).to be_a ::Manticore::ClientStoppedException
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe "auth" do
|
22
|
+
let(:user) { "myuser" }
|
23
|
+
let(:password) { "mypassword" }
|
24
|
+
let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
|
25
|
+
let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
|
26
|
+
|
27
|
+
it "should convert the auth to params" do
|
28
|
+
resp = double("response")
|
29
|
+
allow(resp).to receive(:call)
|
30
|
+
allow(resp).to receive(:code).and_return(200)
|
31
|
+
|
32
|
+
expected_uri = noauth_uri.clone
|
33
|
+
expected_uri.path = "/"
|
34
|
+
|
35
|
+
expect(subject.manticore).to receive(:get).
|
36
|
+
with(expected_uri.to_s, {
|
37
|
+
:headers => {"Content-Type" => "application/json"},
|
38
|
+
:auth => {
|
39
|
+
:user => user,
|
40
|
+
:password => password,
|
41
|
+
:eager => true
|
42
|
+
}
|
43
|
+
}).and_return resp
|
44
|
+
|
45
|
+
subject.perform_request(uri, :get, "/")
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
describe "bad response codes" do
|
50
|
+
let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
51
|
+
|
52
|
+
it "should raise a bad response code error" do
|
53
|
+
resp = double("response")
|
54
|
+
allow(resp).to receive(:call)
|
55
|
+
allow(resp).to receive(:code).and_return(500)
|
56
|
+
allow(resp).to receive(:body).and_return("a body")
|
57
|
+
|
58
|
+
expect(subject.manticore).to receive(:get).
|
59
|
+
with(uri.to_s + "/", anything).
|
60
|
+
and_return(resp)
|
61
|
+
|
62
|
+
uri_with_path = uri.clone
|
63
|
+
uri_with_path.path = "/"
|
64
|
+
|
65
|
+
expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
|
66
|
+
with(resp.code, uri_with_path, nil, resp.body).and_call_original
|
67
|
+
|
68
|
+
expect do
|
69
|
+
subject.perform_request(uri, :get, "/")
|
70
|
+
end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
describe "format_url" do
|
75
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
|
76
|
+
let(:path) { "_bulk" }
|
77
|
+
subject { described_class.new(double("logger"), {}) }
|
78
|
+
|
79
|
+
it "should add the path argument to the uri's path" do
|
80
|
+
expect(subject.format_url(url, path).path).to eq("/path/_bulk")
|
81
|
+
end
|
82
|
+
|
83
|
+
context "when uri contains query parameters" do
|
84
|
+
let(:query_params) { "query=value&key=value2" }
|
85
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
|
86
|
+
let(:formatted) { subject.format_url(url, path)}
|
87
|
+
|
88
|
+
it "should retain query_params after format" do
|
89
|
+
expect(formatted.query).to eq(query_params)
|
90
|
+
end
|
91
|
+
|
92
|
+
context "and the path contains query parameters" do
|
93
|
+
let(:path) { "/special_path?specialParam=123" }
|
94
|
+
|
95
|
+
it "should join the query correctly" do
|
96
|
+
expect(formatted.query).to eq(query_params + "&specialParam=123")
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
context "when the path contains query parameters" do
|
102
|
+
let(:path) { "/special_bulk?pathParam=1"}
|
103
|
+
let(:formatted) { subject.format_url(url, path) }
|
104
|
+
|
105
|
+
it "should add the path correctly" do
|
106
|
+
expect(formatted.path).to eq("#{url.path}special_bulk")
|
107
|
+
expect(subject.remove_double_escaping(formatted.path)).to eq("#{url.path}special_bulk")
|
108
|
+
end
|
109
|
+
|
110
|
+
it "should add the query parameters correctly" do
|
111
|
+
expect(formatted.query).to eq("pathParam=1")
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
context "when uri contains credentials" do
|
116
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
|
117
|
+
let(:formatted) { subject.format_url(url, path) }
|
118
|
+
|
119
|
+
it "should remove credentials after format" do
|
120
|
+
expect(formatted.userinfo).to be_nil
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
context 'when uri contains date math' do
|
125
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
126
|
+
let(:path) { CGI.escape("<logstash-{now/d}-0001>") }
|
127
|
+
let(:formatted) { subject.format_url(url, path) }
|
128
|
+
|
129
|
+
it 'should escape the uri correctly' do
|
130
|
+
expect(subject.remove_double_escaping(formatted.path)).to eq("/%3Clogstash-%7Bnow%2Fd%7D-0001%3E")
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
context 'when uri does not contain date math' do
|
135
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
136
|
+
let(:path) { CGI.escape("logstash-0001") }
|
137
|
+
let(:formatted) { subject.format_url(url, path) }
|
138
|
+
|
139
|
+
it 'should escape the uri correctly' do
|
140
|
+
expect(subject.remove_double_escaping(formatted.path)).to eq("/logstash-0001")
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
describe "integration specs", :integration => true do
|
146
|
+
it "should perform correct tests without error" do
|
147
|
+
resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")
|
148
|
+
expect(resp.code).to eql(200)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
@@ -0,0 +1,501 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/elasticsearch/http_client"
|
3
|
+
require 'cabin'
|
4
|
+
|
5
|
+
describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
6
|
+
let(:logger) { Cabin::Channel.get }
|
7
|
+
let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
|
8
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
9
|
+
let(:options) { {:resurrect_delay => 3, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
|
10
|
+
let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'} ] }
|
11
|
+
let(:license_status) { 'active' }
|
12
|
+
|
13
|
+
subject { described_class.new(logger, adapter, initial_urls, options) }
|
14
|
+
|
15
|
+
let(:manticore_double) { double("manticore a") }
|
16
|
+
before(:each) do
|
17
|
+
response_double = double("manticore response").as_null_object
|
18
|
+
# Allow healtchecks
|
19
|
+
allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
|
20
|
+
allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
|
21
|
+
allow(manticore_double).to receive(:close)
|
22
|
+
|
23
|
+
allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
|
24
|
+
|
25
|
+
allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_version_info)
|
26
|
+
allow(subject.license_checker).to receive(:license_status).and_return(license_status)
|
27
|
+
end
|
28
|
+
|
29
|
+
after do
|
30
|
+
subject.close
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "initialization" do
|
34
|
+
it "should be successful" do
|
35
|
+
expect { subject }.not_to raise_error
|
36
|
+
subject.start
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
describe "the resurrectionist" do
|
41
|
+
before(:each) { subject.start }
|
42
|
+
it "should start the resurrectionist when created" do
|
43
|
+
expect(subject.resurrectionist_alive?).to eql(true)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should attempt to resurrect connections after the ressurrect delay" do
|
47
|
+
expect(subject).to receive(:healthcheck!).once
|
48
|
+
sleep(subject.resurrect_delay + 1)
|
49
|
+
end
|
50
|
+
|
51
|
+
describe "healthcheck url handling" do
|
52
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
53
|
+
let(:success_response) { double("Response", :code => 200) }
|
54
|
+
|
55
|
+
before(:example) do
|
56
|
+
expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
|
57
|
+
expect(url.path).to be_empty
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
context "and not setting healthcheck_path" do
|
62
|
+
it "performs the healthcheck to the root" do
|
63
|
+
expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
|
64
|
+
expect(url.path).to be_empty
|
65
|
+
|
66
|
+
success_response
|
67
|
+
end
|
68
|
+
expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context "and setting healthcheck_path" do
|
73
|
+
let(:healthcheck_path) { "/my/health" }
|
74
|
+
let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
|
75
|
+
it "performs the healthcheck to the healthcheck_path" do
|
76
|
+
expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
|
77
|
+
expect(url.path).to be_empty
|
78
|
+
|
79
|
+
success_response
|
80
|
+
end
|
81
|
+
expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
describe 'resolving the address from Elasticsearch node info' do
|
88
|
+
let(:host) { "node.elastic.co"}
|
89
|
+
let(:ip_address) { "192.168.1.0"}
|
90
|
+
let(:port) { 9200 }
|
91
|
+
|
92
|
+
context 'in Elasticsearch 1.x format' do
|
93
|
+
context 'with host and ip address' do
|
94
|
+
let(:publish_address) { "inet[#{host}/#{ip_address}:#{port}]"}
|
95
|
+
it 'should correctly extract the host' do
|
96
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
|
97
|
+
end
|
98
|
+
end
|
99
|
+
context 'with ip address' do
|
100
|
+
let(:publish_address) { "inet[/#{ip_address}:#{port}]"}
|
101
|
+
it 'should correctly extract the ip address' do
|
102
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
context 'in Elasticsearch 2.x-6.x format' do
|
108
|
+
let(:publish_address) { "#{ip_address}:#{port}"}
|
109
|
+
it 'should correctly extract the ip address' do
|
110
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("//#{ip_address}:#{port}"))
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
context 'in Elasticsearch 7.x'
|
115
|
+
context 'with host and ip address' do
|
116
|
+
let(:publish_address) { "#{host}/#{ip_address}:#{port}"}
|
117
|
+
it 'should correctly extract the host' do
|
118
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
|
119
|
+
end
|
120
|
+
end
|
121
|
+
context 'with ip address' do
|
122
|
+
let(:publish_address) { "#{ip_address}:#{port}"}
|
123
|
+
it 'should correctly extract the ip address' do
|
124
|
+
expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
describe "the sniffer" do
|
130
|
+
before(:each) { subject.start }
|
131
|
+
it "should not start the sniffer by default" do
|
132
|
+
expect(subject.sniffer_alive?).to eql(nil)
|
133
|
+
end
|
134
|
+
|
135
|
+
context "when enabled" do
|
136
|
+
let(:options) { super().merge(:sniffing => true)}
|
137
|
+
|
138
|
+
it "should start the sniffer" do
|
139
|
+
expect(subject.sniffer_alive?).to eql(true)
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
describe "closing" do
|
145
|
+
before do
|
146
|
+
subject.start
|
147
|
+
# Simulate a single in use connection on the first check of this
|
148
|
+
allow(adapter).to receive(:close).and_call_original
|
149
|
+
allow(subject).to receive(:wait_for_in_use_connections).and_call_original
|
150
|
+
allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
|
151
|
+
allow(subject).to receive(:start)
|
152
|
+
subject.close
|
153
|
+
end
|
154
|
+
|
155
|
+
it "should close the adapter" do
|
156
|
+
expect(adapter).to have_received(:close)
|
157
|
+
end
|
158
|
+
|
159
|
+
it "should stop the resurrectionist" do
|
160
|
+
expect(subject.resurrectionist_alive?).to eql(false)
|
161
|
+
end
|
162
|
+
|
163
|
+
it "should stop the sniffer" do
|
164
|
+
# If no sniffer (the default) returns nil
|
165
|
+
expect(subject.sniffer_alive?).to be_falsey
|
166
|
+
end
|
167
|
+
|
168
|
+
it "should wait for in use connections to terminate" do
|
169
|
+
expect(subject).to have_received(:wait_for_in_use_connections).once
|
170
|
+
expect(subject).to have_received(:in_use_connections).twice
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
class MockResponse
|
175
|
+
attr_reader :code, :headers
|
176
|
+
|
177
|
+
def initialize(code = 200, body = nil, headers = {})
|
178
|
+
@code = code
|
179
|
+
@body = body
|
180
|
+
@headers = headers
|
181
|
+
end
|
182
|
+
|
183
|
+
def body
|
184
|
+
@body.to_json
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
describe "connection management" do
|
189
|
+
before(:each) { subject.start }
|
190
|
+
context "with only one URL in the list" do
|
191
|
+
it "should use the only URL in 'with_connection'" do
|
192
|
+
subject.with_connection do |c|
|
193
|
+
expect(c).to eq(initial_urls.first)
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
context "with multiple URLs in the list" do
|
199
|
+
let(:version_ok) do
|
200
|
+
MockResponse.new(200, {"tagline" => "You Know, for Search",
|
201
|
+
"version" => {
|
202
|
+
"number" => '7.13.0',
|
203
|
+
"build_flavor" => 'default'}
|
204
|
+
})
|
205
|
+
end
|
206
|
+
let(:success_response) { double("head_req", :code => 200)}
|
207
|
+
|
208
|
+
before :each do
|
209
|
+
allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil).and_return(success_response)
|
210
|
+
allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
|
211
|
+
end
|
212
|
+
let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
|
213
|
+
|
214
|
+
it "should minimize the number of connections to a single URL" do
|
215
|
+
connected_urls = []
|
216
|
+
|
217
|
+
# If we make 2x the number requests as we have URLs we should
|
218
|
+
# connect to each URL exactly 2 times
|
219
|
+
(initial_urls.size*2).times do
|
220
|
+
u, meta = subject.get_connection
|
221
|
+
connected_urls << u
|
222
|
+
end
|
223
|
+
|
224
|
+
connected_urls.each {|u| subject.return_connection(u) }
|
225
|
+
initial_urls.each do |url|
|
226
|
+
conn_count = connected_urls.select {|u| u == url}.size
|
227
|
+
expect(conn_count).to eql(2)
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
it "should correctly resurrect the dead" do
|
232
|
+
u,m = subject.get_connection
|
233
|
+
|
234
|
+
# The resurrectionist will call this to check on the backend
|
235
|
+
response = double("response", :code => 200)
|
236
|
+
expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
|
237
|
+
|
238
|
+
subject.return_connection(u)
|
239
|
+
subject.mark_dead(u, Exception.new)
|
240
|
+
|
241
|
+
expect(subject.url_meta(u)[:state]).to eql(:dead)
|
242
|
+
sleep subject.resurrect_delay + 1
|
243
|
+
expect(subject.url_meta(u)[:state]).to eql(:alive)
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
describe "version tracking" do
|
249
|
+
let(:initial_urls) { [
|
250
|
+
::LogStash::Util::SafeURI.new("http://somehost:9200"),
|
251
|
+
::LogStash::Util::SafeURI.new("http://otherhost:9201")
|
252
|
+
] }
|
253
|
+
|
254
|
+
let(:valid_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
|
255
|
+
"version" => {
|
256
|
+
"number" => '7.13.0',
|
257
|
+
"build_flavor" => 'default'}
|
258
|
+
}) }
|
259
|
+
|
260
|
+
before(:each) do
|
261
|
+
allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
|
262
|
+
subject.start
|
263
|
+
end
|
264
|
+
|
265
|
+
it "picks the largest major version" do
|
266
|
+
expect(subject.maximum_seen_major_version).to eq(0)
|
267
|
+
end
|
268
|
+
|
269
|
+
context "if there are nodes with multiple major versions" do
|
270
|
+
let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'}, { "number" => '6.0.0', "build_flavor" => 'default'} ] }
|
271
|
+
it "picks the largest major version" do
|
272
|
+
expect(subject.maximum_seen_major_version).to eq(6)
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
|
278
|
+
describe "build flavour tracking" do
|
279
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://somehost:9200")] }
|
280
|
+
|
281
|
+
let(:es_version_info) { [ { "number" => '8.9.0', "build_flavor" => "serverless" } ] }
|
282
|
+
|
283
|
+
let(:valid_response) { MockResponse.new(200,
|
284
|
+
{"tagline" => "You Know, for Search",
|
285
|
+
"version" => {
|
286
|
+
"number" => '8.9.0',
|
287
|
+
"build_flavor" => LogStash::Outputs::ElasticSearch::HttpClient::Pool::BUILD_FLAVOUR_SERVERLESS} },
|
288
|
+
{ "X-Elastic-Product" => "Elasticsearch" }
|
289
|
+
) }
|
290
|
+
|
291
|
+
before(:each) do
|
292
|
+
allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
|
293
|
+
subject.start
|
294
|
+
end
|
295
|
+
|
296
|
+
it "picks the build flavour" do
|
297
|
+
expect(subject.serverless?).to be_truthy
|
298
|
+
end
|
299
|
+
end
|
300
|
+
|
301
|
+
describe "license checking" do
|
302
|
+
before(:each) do
|
303
|
+
allow(subject).to receive(:health_check_request)
|
304
|
+
allow(subject).to receive(:elasticsearch?).and_return(true)
|
305
|
+
end
|
306
|
+
|
307
|
+
let(:options) do
|
308
|
+
super().merge(:license_checker => license_checker)
|
309
|
+
end
|
310
|
+
|
311
|
+
context 'when LicenseChecker#acceptable_license? returns false' do
|
312
|
+
let(:license_checker) { double('LicenseChecker', :appropriate_license? => false) }
|
313
|
+
|
314
|
+
it 'does not mark the URL as active' do
|
315
|
+
subject.update_initial_urls
|
316
|
+
expect(subject.alive_urls_count).to eq(0)
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
context 'when LicenseChecker#acceptable_license? returns true' do
|
321
|
+
let(:license_checker) { double('LicenseChecker', :appropriate_license? => true) }
|
322
|
+
|
323
|
+
it 'marks the URL as active' do
|
324
|
+
subject.update_initial_urls
|
325
|
+
expect(subject.alive_urls_count).to eq(1)
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
330
|
+
# TODO: extract to ElasticSearchOutputLicenseChecker unit spec
|
331
|
+
describe "license checking with ElasticSearchOutputLicenseChecker" do
|
332
|
+
let(:options) do
|
333
|
+
super().merge(:license_checker => LogStash::Outputs::ElasticSearch::LicenseChecker.new(logger))
|
334
|
+
end
|
335
|
+
|
336
|
+
before(:each) do
|
337
|
+
allow(subject).to receive(:health_check_request)
|
338
|
+
allow(subject).to receive(:elasticsearch?).and_return(true)
|
339
|
+
end
|
340
|
+
|
341
|
+
context "if ES doesn't return a valid license" do
|
342
|
+
let(:license_status) { nil }
|
343
|
+
|
344
|
+
it "marks the url as dead" do
|
345
|
+
subject.update_initial_urls
|
346
|
+
expect(subject.alive_urls_count).to eq(0)
|
347
|
+
end
|
348
|
+
|
349
|
+
it "logs a warning" do
|
350
|
+
expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
|
351
|
+
subject.update_initial_urls
|
352
|
+
end
|
353
|
+
end
|
354
|
+
|
355
|
+
context "if ES returns a valid license" do
|
356
|
+
let(:license_status) { 'active' }
|
357
|
+
|
358
|
+
it "marks the url as active" do
|
359
|
+
subject.update_initial_urls
|
360
|
+
expect(subject.alive_urls_count).to eq(1)
|
361
|
+
end
|
362
|
+
|
363
|
+
it "does not log a warning" do
|
364
|
+
expect(subject.license_checker).to_not receive(:warn_no_license)
|
365
|
+
expect(subject.license_checker).to_not receive(:warn_invalid_license)
|
366
|
+
subject.update_initial_urls
|
367
|
+
end
|
368
|
+
end
|
369
|
+
|
370
|
+
context "if ES returns an invalid license" do
|
371
|
+
let(:license_status) { 'invalid' }
|
372
|
+
|
373
|
+
it "marks the url as active" do
|
374
|
+
subject.update_initial_urls
|
375
|
+
expect(subject.alive_urls_count).to eq(1)
|
376
|
+
end
|
377
|
+
|
378
|
+
it "logs a warning" do
|
379
|
+
expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
|
380
|
+
subject.update_initial_urls
|
381
|
+
end
|
382
|
+
end
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
describe "#elasticsearch?" do
|
387
|
+
let(:logger) { Cabin::Channel.get }
|
388
|
+
let(:adapter) { double("Manticore Adapter") }
|
389
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
390
|
+
let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
|
391
|
+
let(:es_version_info) { [{ "number" => '0.0.0', "build_flavor" => 'default'}] }
|
392
|
+
let(:license_status) { 'active' }
|
393
|
+
|
394
|
+
subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
|
395
|
+
|
396
|
+
let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
|
397
|
+
|
398
|
+
context "in case HTTP error code" do
|
399
|
+
it "should fail for 401" do
|
400
|
+
allow(adapter).to receive(:perform_request)
|
401
|
+
.with(anything, :get, "/", anything, anything)
|
402
|
+
.and_return(MockResponse.new(401))
|
403
|
+
|
404
|
+
expect(subject.elasticsearch?(url)).to be false
|
405
|
+
end
|
406
|
+
|
407
|
+
it "should fail for 403" do
|
408
|
+
allow(adapter).to receive(:perform_request)
|
409
|
+
.with(anything, :get, "/", anything, anything)
|
410
|
+
.and_return(status: 403)
|
411
|
+
expect(subject.elasticsearch?(url)).to be false
|
412
|
+
end
|
413
|
+
end
|
414
|
+
|
415
|
+
context "when connecting to a cluster which reply without 'version' field" do
|
416
|
+
it "should fail" do
|
417
|
+
allow(adapter).to receive(:perform_request)
|
418
|
+
.with(anything, :get, "/", anything, anything)
|
419
|
+
.and_return(body: {"field" => "funky.com"}.to_json)
|
420
|
+
expect(subject.elasticsearch?(url)).to be false
|
421
|
+
end
|
422
|
+
end
|
423
|
+
|
424
|
+
context "when connecting to a cluster with version < 6.0.0" do
|
425
|
+
it "should fail" do
|
426
|
+
allow(adapter).to receive(:perform_request)
|
427
|
+
.with(anything, :get, "/", anything, anything)
|
428
|
+
.and_return(200, {"version" => { "number" => "5.0.0"}}.to_json)
|
429
|
+
expect(subject.elasticsearch?(url)).to be false
|
430
|
+
end
|
431
|
+
end
|
432
|
+
|
433
|
+
context "when connecting to a cluster with version in [6.0.0..7.0.0)" do
|
434
|
+
it "must be successful with valid 'tagline'" do
|
435
|
+
allow(adapter).to receive(:perform_request)
|
436
|
+
.with(anything, :get, "/", anything, anything)
|
437
|
+
.and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"}))
|
438
|
+
expect(subject.elasticsearch?(url)).to be true
|
439
|
+
end
|
440
|
+
|
441
|
+
it "should fail if invalid 'tagline'" do
|
442
|
+
allow(adapter).to receive(:perform_request)
|
443
|
+
.with(anything, :get, "/", anything, anything)
|
444
|
+
.and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"}))
|
445
|
+
expect(subject.elasticsearch?(url)).to be false
|
446
|
+
end
|
447
|
+
|
448
|
+
it "should fail if 'tagline' is not present" do
|
449
|
+
allow(adapter).to receive(:perform_request)
|
450
|
+
.with(anything, :get, "/", anything, anything)
|
451
|
+
.and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}}))
|
452
|
+
expect(subject.elasticsearch?(url)).to be false
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
context "when connecting to a cluster with version in [7.0.0..7.14.0)" do
|
457
|
+
it "must be successful is 'build_flavor' is 'default' and tagline is correct" do
|
458
|
+
allow(adapter).to receive(:perform_request)
|
459
|
+
.with(anything, :get, "/", anything, anything)
|
460
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"}))
|
461
|
+
expect(subject.elasticsearch?(url)).to be true
|
462
|
+
end
|
463
|
+
|
464
|
+
it "should fail if 'build_flavor' is not 'default' and tagline is correct" do
|
465
|
+
allow(adapter).to receive(:perform_request)
|
466
|
+
.with(anything, :get, "/", anything, anything)
|
467
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"}))
|
468
|
+
expect(subject.elasticsearch?(url)).to be false
|
469
|
+
end
|
470
|
+
|
471
|
+
it "should fail if 'build_flavor' is not present and tagline is correct" do
|
472
|
+
allow(adapter).to receive(:perform_request)
|
473
|
+
.with(anything, :get, "/", anything, anything)
|
474
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"}))
|
475
|
+
expect(subject.elasticsearch?(url)).to be false
|
476
|
+
end
|
477
|
+
end
|
478
|
+
|
479
|
+
context "when connecting to a cluster with version >= 7.14.0" do
|
480
|
+
it "should fail if 'X-elastic-product' header is not present" do
|
481
|
+
allow(adapter).to receive(:perform_request)
|
482
|
+
.with(anything, :get, "/", anything, anything)
|
483
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}))
|
484
|
+
expect(subject.elasticsearch?(url)).to be false
|
485
|
+
end
|
486
|
+
|
487
|
+
it "should fail if 'X-elastic-product' header is present but with bad value" do
|
488
|
+
allow(adapter).to receive(:perform_request)
|
489
|
+
.with(anything, :get, "/", anything, anything)
|
490
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'}))
|
491
|
+
expect(subject.elasticsearch?(url)).to be false
|
492
|
+
end
|
493
|
+
|
494
|
+
it "must be successful when 'X-elastic-product' header is present with 'Elasticsearch' value" do
|
495
|
+
allow(adapter).to receive(:perform_request)
|
496
|
+
.with(anything, :get, "/", anything, anything)
|
497
|
+
.and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'}))
|
498
|
+
expect(subject.elasticsearch?(url)).to be true
|
499
|
+
end
|
500
|
+
end
|
501
|
+
end
|