logstash-output-elasticsearch 11.22.17-java → 12.0.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -20
- data/README.md +1 -3
- data/docs/index.asciidoc +21 -112
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +4 -1
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +22 -12
- data/lib/logstash/outputs/elasticsearch/http_client.rb +31 -44
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +2 -31
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +1 -16
- data/lib/logstash/outputs/elasticsearch.rb +0 -47
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +13 -30
- data/lib/logstash/plugin_mixins/elasticsearch/common.rb +1 -11
- data/logstash-output-elasticsearch.gemspec +2 -2
- data/spec/es_spec_helper.rb +22 -34
- data/spec/fixtures/test_certs/GENERATED_AT +1 -1
- data/spec/fixtures/test_certs/ca.crt +27 -30
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -1
- data/spec/fixtures/test_certs/renew.sh +3 -2
- data/spec/fixtures/test_certs/test.crt +28 -29
- data/spec/fixtures/test_certs/test.der.sha256 +1 -1
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +1 -3
- data/spec/integration/outputs/delete_spec.rb +4 -4
- data/spec/integration/outputs/ilm_spec.rb +12 -18
- data/spec/integration/outputs/index_spec.rb +2 -12
- data/spec/integration/outputs/index_version_spec.rb +7 -7
- data/spec/integration/outputs/metrics_spec.rb +1 -4
- data/spec/integration/outputs/painless_update_spec.rb +10 -11
- data/spec/integration/outputs/unsupported_actions_spec.rb +10 -15
- data/spec/integration/outputs/update_spec.rb +9 -11
- data/spec/spec_helper.rb +0 -8
- data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
- data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +0 -77
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +0 -71
- data/spec/unit/outputs/elasticsearch_spec.rb +3 -112
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +22 -0
- data/spec/unit/outputs/error_whitelist_spec.rb +1 -0
- metadata +25 -19
- data/version +0 -1
|
@@ -5,10 +5,7 @@ describe "metrics", :integration => true do
|
|
|
5
5
|
require "logstash/outputs/elasticsearch"
|
|
6
6
|
settings = {
|
|
7
7
|
"manage_template" => false,
|
|
8
|
-
"hosts" => "#{get_host_port()}"
|
|
9
|
-
# write data to a random non templated index to ensure the bulk partially fails
|
|
10
|
-
# don't use streams like "logs-*" as those have failure stores enabled, causing the bulk to succeed instead
|
|
11
|
-
"index" => "custom_index_#{rand(10000)}"
|
|
8
|
+
"hosts" => "#{get_host_port()}"
|
|
12
9
|
}
|
|
13
10
|
plugin = LogStash::Outputs::ElasticSearch.new(settings)
|
|
14
11
|
end
|
|
@@ -22,12 +22,11 @@ describe "Update actions using painless scripts", :integration => true, :update_
|
|
|
22
22
|
# This can fail if there are no indexes, ignore failure.
|
|
23
23
|
@es.indices.delete(:index => "*") rescue nil
|
|
24
24
|
@es.index(
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
})
|
|
25
|
+
:index => 'logstash-update',
|
|
26
|
+
:type => doc_type,
|
|
27
|
+
:id => "123",
|
|
28
|
+
:body => { :message => 'Test', :counter => 1 }
|
|
29
|
+
)
|
|
31
30
|
@es.indices.refresh
|
|
32
31
|
end
|
|
33
32
|
|
|
@@ -47,7 +46,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
|
|
|
47
46
|
subject = get_es_output(plugin_parameters)
|
|
48
47
|
subject.register
|
|
49
48
|
subject.multi_receive([LogStash::Event.new("count" => 4 )])
|
|
50
|
-
r = @es.get(:index => 'logstash-update', :id => "123", :refresh => true)
|
|
49
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
|
51
50
|
expect(r["_source"]["counter"]).to eq(5)
|
|
52
51
|
end
|
|
53
52
|
end
|
|
@@ -58,7 +57,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
|
|
|
58
57
|
subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
|
59
58
|
subject.register
|
|
60
59
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
61
|
-
r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true)
|
|
60
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
|
62
61
|
expect(r["_source"]["message"]).to eq('upsert message')
|
|
63
62
|
end
|
|
64
63
|
|
|
@@ -66,7 +65,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
|
|
|
66
65
|
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
|
|
67
66
|
subject.register
|
|
68
67
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
69
|
-
r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true)
|
|
68
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
|
70
69
|
expect(r["_source"]["message"]).to eq('sample message here')
|
|
71
70
|
end
|
|
72
71
|
|
|
@@ -83,7 +82,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
|
|
|
83
82
|
subject.register
|
|
84
83
|
|
|
85
84
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
86
|
-
r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true)
|
|
85
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
|
87
86
|
expect(r["_source"]["message"]).to eq('upsert message')
|
|
88
87
|
end
|
|
89
88
|
|
|
@@ -92,7 +91,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
|
|
|
92
91
|
subject.register
|
|
93
92
|
subject.multi_receive([LogStash::Event.new("counter" => 1)])
|
|
94
93
|
@es.indices.refresh
|
|
95
|
-
r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true)
|
|
94
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
|
96
95
|
expect(r["_source"]["counter"]).to eq(1)
|
|
97
96
|
end
|
|
98
97
|
end
|
|
@@ -27,21 +27,16 @@ describe "Unsupported actions testing...", :integration => true do
|
|
|
27
27
|
@es.indices.delete(:index => "*") rescue nil
|
|
28
28
|
# index single doc for update purpose
|
|
29
29
|
@es.index(
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
:refresh => true
|
|
35
|
-
}
|
|
30
|
+
:index => INDEX,
|
|
31
|
+
:type => doc_type,
|
|
32
|
+
:id => "2",
|
|
33
|
+
:body => { :message => 'Test to doc indexing', :counter => 1 }
|
|
36
34
|
)
|
|
37
|
-
|
|
38
35
|
@es.index(
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
:refresh => true
|
|
44
|
-
}
|
|
36
|
+
:index => INDEX,
|
|
37
|
+
:type => doc_type,
|
|
38
|
+
:id => "3",
|
|
39
|
+
:body => { :message => 'Test to doc deletion', :counter => 2 }
|
|
45
40
|
)
|
|
46
41
|
@es.indices.refresh
|
|
47
42
|
end
|
|
@@ -68,12 +63,12 @@ describe "Unsupported actions testing...", :integration => true do
|
|
|
68
63
|
rejected_events = events.select { |event| !index_or_update.call(event) }
|
|
69
64
|
|
|
70
65
|
indexed_events.each do |event|
|
|
71
|
-
response = @es.get(:index => INDEX, :id => event.get("doc_id"), :refresh => true)
|
|
66
|
+
response = @es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)
|
|
72
67
|
expect(response['_source']['message']).to eq(event.get("message"))
|
|
73
68
|
end
|
|
74
69
|
|
|
75
70
|
rejected_events.each do |event|
|
|
76
|
-
expect {@es.get(:index => INDEX, :id => event.get("doc_id"), :refresh => true)}.to raise_error(
|
|
71
|
+
expect {@es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
|
|
77
72
|
end
|
|
78
73
|
end
|
|
79
74
|
end
|
|
@@ -22,12 +22,10 @@ describe "Update actions without scripts", :integration => true do
|
|
|
22
22
|
# This can fail if there are no indexes, ignore failure.
|
|
23
23
|
@es.indices.delete(:index => "*") rescue nil
|
|
24
24
|
@es.index(
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
:refresh => true
|
|
30
|
-
}
|
|
25
|
+
:index => 'logstash-update',
|
|
26
|
+
:type => doc_type,
|
|
27
|
+
:id => "123",
|
|
28
|
+
:body => { :message => 'Test', :counter => 1 }
|
|
31
29
|
)
|
|
32
30
|
@es.indices.refresh
|
|
33
31
|
end
|
|
@@ -42,14 +40,14 @@ describe "Update actions without scripts", :integration => true do
|
|
|
42
40
|
subject = get_es_output({ 'document_id' => "456" } )
|
|
43
41
|
subject.register
|
|
44
42
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
45
|
-
expect {@es.get(:index => 'logstash-update', :id =>
|
|
43
|
+
expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
|
|
46
44
|
end
|
|
47
45
|
|
|
48
46
|
it "should update existing document" do
|
|
49
47
|
subject = get_es_output({ 'document_id' => "123" })
|
|
50
48
|
subject.register
|
|
51
49
|
subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
|
|
52
|
-
r = @es.get(:index => 'logstash-update', :id =>
|
|
50
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
|
53
51
|
expect(r["_source"]["message"]).to eq('updated message here')
|
|
54
52
|
end
|
|
55
53
|
|
|
@@ -59,7 +57,7 @@ describe "Update actions without scripts", :integration => true do
|
|
|
59
57
|
subject = get_es_output({ 'document_id' => "123" })
|
|
60
58
|
subject.register
|
|
61
59
|
subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
|
|
62
|
-
r = @es.get(:index => 'logstash-update', :id =>
|
|
60
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
|
|
63
61
|
expect(r["_source"]["data"]).to eq('updated message here')
|
|
64
62
|
expect(r["_source"]["message"]).to eq('foo')
|
|
65
63
|
end
|
|
@@ -96,7 +94,7 @@ describe "Update actions without scripts", :integration => true do
|
|
|
96
94
|
subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
|
|
97
95
|
subject.register
|
|
98
96
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
99
|
-
r = @es.get(:index => 'logstash-update', :id =>
|
|
97
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
|
100
98
|
expect(r["_source"]["message"]).to eq('upsert message')
|
|
101
99
|
end
|
|
102
100
|
|
|
@@ -104,7 +102,7 @@ describe "Update actions without scripts", :integration => true do
|
|
|
104
102
|
subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
|
|
105
103
|
subject.register
|
|
106
104
|
subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
|
|
107
|
-
r = @es.get(:index => 'logstash-update', :id =>
|
|
105
|
+
r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
|
|
108
106
|
expect(r["_source"]["message"]).to eq('sample message here')
|
|
109
107
|
end
|
|
110
108
|
|
data/spec/spec_helper.rb
CHANGED
|
@@ -8,11 +8,3 @@ end
|
|
|
8
8
|
RSpec.configure do |config|
|
|
9
9
|
config.include LogStash::Outputs::ElasticSearch::SpecHelper
|
|
10
10
|
end
|
|
11
|
-
|
|
12
|
-
# remove once plugin starts consuming elasticsearch-ruby v8 client
|
|
13
|
-
def elastic_ruby_v8_client_available?
|
|
14
|
-
Elasticsearch::Transport
|
|
15
|
-
false
|
|
16
|
-
rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
|
|
17
|
-
true
|
|
18
|
-
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
2
|
+
# or more contributor license agreements. Licensed under the Elastic License;
|
|
3
|
+
# you may not use this file except in compliance with the Elastic License.
|
|
4
|
+
|
|
5
|
+
module Elasticsearch
|
|
6
|
+
module API
|
|
7
|
+
module Actions
|
|
8
|
+
|
|
9
|
+
# Retrieve the list of index lifecycle management policies
|
|
10
|
+
def get_alias(arguments={})
|
|
11
|
+
method = HTTP_GET
|
|
12
|
+
path = Utils.__pathify '_alias', Utils.__escape(arguments[:name])
|
|
13
|
+
params = {}
|
|
14
|
+
perform_request(method, path, params, nil).body
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
2
|
+
# or more contributor license agreements. Licensed under the Elastic License;
|
|
3
|
+
# you may not use this file except in compliance with the Elastic License.
|
|
4
|
+
|
|
5
|
+
module Elasticsearch
|
|
6
|
+
module API
|
|
7
|
+
module Actions
|
|
8
|
+
|
|
9
|
+
# @option arguments [String] :name The name of the alias (*Required*)
|
|
10
|
+
# @option arguments [Hash] :The alias definition(*Required*)
|
|
11
|
+
|
|
12
|
+
def put_alias(arguments={})
|
|
13
|
+
raise ArgumentError, "Required argument 'name' missing" unless arguments[:name]
|
|
14
|
+
raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
|
|
15
|
+
method = HTTP_PUT
|
|
16
|
+
path = Utils.__pathify Utils.__escape(arguments[:name])
|
|
17
|
+
|
|
18
|
+
params = Utils.__validate_and_extract_params arguments
|
|
19
|
+
body = arguments[:body]
|
|
20
|
+
perform_request(method, path, params, body.to_json).body
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -270,83 +270,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
|
270
270
|
|
|
271
271
|
end
|
|
272
272
|
end
|
|
273
|
-
context "the 'user-agent' header" do
|
|
274
|
-
let(:pool) { double("pool") }
|
|
275
|
-
let(:compression_level) { 6 }
|
|
276
|
-
let(:base_options) { super().merge( :client_settings => {:compression_level => compression_level}) }
|
|
277
|
-
let(:actions) { [
|
|
278
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message_1"=> message_1}],
|
|
279
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message_2"=> message_2}],
|
|
280
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message_3"=> message_3}],
|
|
281
|
-
]}
|
|
282
|
-
let(:message_1) { "hello" }
|
|
283
|
-
let(:message_2_size) { 1_000 }
|
|
284
|
-
let(:message_2) { SecureRandom.alphanumeric(message_2_size / 2 ) * 2 }
|
|
285
|
-
let(:message_3_size) { 1_000 }
|
|
286
|
-
let(:message_3) { "m" * message_3_size }
|
|
287
|
-
let(:messages_size) { message_1.size + message_2.size + message_3.size }
|
|
288
|
-
let(:action_overhead) { 42 + 16 + 2 } # header plus doc key size plus new line overhead per action
|
|
289
|
-
|
|
290
|
-
let(:response) do
|
|
291
|
-
response = double("response")
|
|
292
|
-
allow(response).to receive(:code).and_return(response)
|
|
293
|
-
allow(response).to receive(:body).and_return({"errors" => false}.to_json)
|
|
294
|
-
response
|
|
295
|
-
end
|
|
296
|
-
|
|
297
|
-
before(:each) do
|
|
298
|
-
subject.instance_variable_set("@pool", pool)
|
|
299
|
-
end
|
|
300
|
-
|
|
301
|
-
it "carries bulk request's uncompressed size" do
|
|
302
|
-
expect(pool).to receive(:post) do |path, params, body|
|
|
303
|
-
headers = params.fetch(:headers, {})
|
|
304
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("3")
|
|
305
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
|
|
306
|
-
end.and_return(response)
|
|
307
|
-
|
|
308
|
-
subject.send(:bulk, actions)
|
|
309
|
-
end
|
|
310
|
-
context "without compression" do
|
|
311
|
-
let(:compression_level) { 0 }
|
|
312
|
-
it "carries bulk request's uncompressed size" do
|
|
313
|
-
expect(pool).to receive(:post) do |path, params, body|
|
|
314
|
-
headers = params.fetch(:headers, {})
|
|
315
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("3")
|
|
316
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
|
|
317
|
-
end.and_return(response)
|
|
318
|
-
subject.send(:bulk, actions)
|
|
319
|
-
end
|
|
320
|
-
end
|
|
321
|
-
|
|
322
|
-
context "with compressed messages over 20MB" do
|
|
323
|
-
let(:message_2_size) { 21_000_000 }
|
|
324
|
-
it "carries bulk request's uncompressed size" do
|
|
325
|
-
# only the first, tiny, message is sent first
|
|
326
|
-
expect(pool).to receive(:post) do |path, params, body|
|
|
327
|
-
headers = params.fetch(:headers, {})
|
|
328
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_1.size + action_overhead).to_s
|
|
329
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("1")
|
|
330
|
-
end.and_return(response)
|
|
331
|
-
|
|
332
|
-
# huge message_2 is sent afterwards alone
|
|
333
|
-
expect(pool).to receive(:post) do |path, params, body|
|
|
334
|
-
headers = params.fetch(:headers, {})
|
|
335
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_2.size + action_overhead).to_s
|
|
336
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("1")
|
|
337
|
-
end.and_return(response)
|
|
338
|
-
|
|
339
|
-
# finally medium message_3 is sent alone as well
|
|
340
|
-
expect(pool).to receive(:post) do |path, params, body|
|
|
341
|
-
headers = params.fetch(:headers, {})
|
|
342
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_3.size + action_overhead).to_s
|
|
343
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("1")
|
|
344
|
-
end.and_return(response)
|
|
345
|
-
|
|
346
|
-
subject.send(:bulk, actions)
|
|
347
|
-
end
|
|
348
|
-
end
|
|
349
|
-
end
|
|
350
273
|
end
|
|
351
274
|
|
|
352
275
|
describe "sniffing" do
|
|
@@ -186,75 +186,4 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
|
|
|
186
186
|
|
|
187
187
|
end
|
|
188
188
|
end
|
|
189
|
-
|
|
190
|
-
describe ".strip_serverless_incompatible_settings" do
|
|
191
|
-
context "with composable template (template.settings)" do
|
|
192
|
-
let(:template) do
|
|
193
|
-
{
|
|
194
|
-
"template" => {
|
|
195
|
-
"settings" => {
|
|
196
|
-
"index.refresh_interval" => "5s",
|
|
197
|
-
"number_of_shards" => 1,
|
|
198
|
-
"number_of_replicas" => 0
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
end
|
|
203
|
-
|
|
204
|
-
it "removes number_of_shards and number_of_replicas" do
|
|
205
|
-
described_class.strip_serverless_incompatible_settings(template)
|
|
206
|
-
expect(template["template"]["settings"]).not_to have_key("number_of_shards")
|
|
207
|
-
expect(template["template"]["settings"]).not_to have_key("number_of_replicas")
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
it "preserves compatible settings" do
|
|
211
|
-
described_class.strip_serverless_incompatible_settings(template)
|
|
212
|
-
expect(template["template"]["settings"]["index.refresh_interval"]).to eq("5s")
|
|
213
|
-
end
|
|
214
|
-
end
|
|
215
|
-
|
|
216
|
-
context "with index-prefixed setting keys" do
|
|
217
|
-
let(:template) do
|
|
218
|
-
{
|
|
219
|
-
"template" => {
|
|
220
|
-
"settings" => {
|
|
221
|
-
"index.number_of_shards" => 1,
|
|
222
|
-
"index.number_of_replicas" => 0
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
end
|
|
227
|
-
|
|
228
|
-
it "removes index-prefixed variants" do
|
|
229
|
-
described_class.strip_serverless_incompatible_settings(template)
|
|
230
|
-
expect(template["template"]["settings"]).not_to have_key("index.number_of_shards")
|
|
231
|
-
expect(template["template"]["settings"]).not_to have_key("index.number_of_replicas")
|
|
232
|
-
end
|
|
233
|
-
end
|
|
234
|
-
|
|
235
|
-
context "with legacy template (top-level settings)" do
|
|
236
|
-
let(:template) do
|
|
237
|
-
{
|
|
238
|
-
"settings" => {
|
|
239
|
-
"index.refresh_interval" => "5s",
|
|
240
|
-
"number_of_shards" => 1
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
end
|
|
244
|
-
|
|
245
|
-
it "removes incompatible settings from top-level settings" do
|
|
246
|
-
described_class.strip_serverless_incompatible_settings(template)
|
|
247
|
-
expect(template["settings"]).not_to have_key("number_of_shards")
|
|
248
|
-
expect(template["settings"]["index.refresh_interval"]).to eq("5s")
|
|
249
|
-
end
|
|
250
|
-
end
|
|
251
|
-
|
|
252
|
-
context "with no settings" do
|
|
253
|
-
let(:template) { { "mappings" => {} } }
|
|
254
|
-
|
|
255
|
-
it "does not raise" do
|
|
256
|
-
expect { described_class.strip_serverless_incompatible_settings(template) }.not_to raise_error
|
|
257
|
-
end
|
|
258
|
-
end
|
|
259
|
-
end
|
|
260
189
|
end
|
|
@@ -777,7 +777,7 @@ describe LogStash::Outputs::ElasticSearch do
|
|
|
777
777
|
end
|
|
778
778
|
|
|
779
779
|
before(:each) do
|
|
780
|
-
allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)
|
|
780
|
+
allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
|
|
781
781
|
expect( stream.string ).to include '"foo":"bar1"'
|
|
782
782
|
expect( stream.string ).to include '"foo":"bar2"'
|
|
783
783
|
end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely
|
|
@@ -908,9 +908,6 @@ describe LogStash::Outputs::ElasticSearch do
|
|
|
908
908
|
|
|
909
909
|
before(:each) do
|
|
910
910
|
allow(elasticsearch_output_instance.client).to receive(:logger).and_return(logger_stub)
|
|
911
|
-
# Stub the output plugin's own @logger to prevent submit()/retrying_submit()
|
|
912
|
-
# from logging the full 15MB action payload on each retry, which floods stdout.
|
|
913
|
-
elasticsearch_output_instance.instance_variable_set(:@logger, logger_stub)
|
|
914
911
|
|
|
915
912
|
allow(elasticsearch_output_instance.client).to receive(:bulk).and_call_original
|
|
916
913
|
|
|
@@ -918,12 +915,7 @@ describe LogStash::Outputs::ElasticSearch do
|
|
|
918
915
|
allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body|
|
|
919
916
|
if body.length > max_bytes
|
|
920
917
|
max_bytes *= 2 # ensure a successful retry
|
|
921
|
-
|
|
922
|
-
413,
|
|
923
|
-
"test-url",
|
|
924
|
-
body,
|
|
925
|
-
""
|
|
926
|
-
)
|
|
918
|
+
double("Response", :code => 413, :body => "")
|
|
927
919
|
else
|
|
928
920
|
double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}')
|
|
929
921
|
end
|
|
@@ -1133,81 +1125,6 @@ describe LogStash::Outputs::ElasticSearch do
|
|
|
1133
1125
|
end
|
|
1134
1126
|
end
|
|
1135
1127
|
|
|
1136
|
-
describe "SSL deprecated settings" do
|
|
1137
|
-
let(:base_options) { {"ssl" => "true"} }
|
|
1138
|
-
|
|
1139
|
-
context "with client certificate" do
|
|
1140
|
-
let(:do_register) { true }
|
|
1141
|
-
let(:cacert) { Stud::Temporary.file.path }
|
|
1142
|
-
let(:options) { base_options.merge(
|
|
1143
|
-
"cacert" => cacert,
|
|
1144
|
-
"ssl_certificate_verification" => false
|
|
1145
|
-
) }
|
|
1146
|
-
|
|
1147
|
-
after :each do
|
|
1148
|
-
File.delete(cacert)
|
|
1149
|
-
end
|
|
1150
|
-
|
|
1151
|
-
it "should map new configs into params" do
|
|
1152
|
-
expect(subject.params).to match hash_including(
|
|
1153
|
-
"ssl_enabled" => true,
|
|
1154
|
-
"ssl_verification_mode" => "none",
|
|
1155
|
-
"ssl_certificate_authorities" => [cacert]
|
|
1156
|
-
)
|
|
1157
|
-
end
|
|
1158
|
-
|
|
1159
|
-
it "should set new configs variables" do
|
|
1160
|
-
expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
|
|
1161
|
-
expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("none")
|
|
1162
|
-
expect(subject.instance_variable_get(:@ssl_certificate_authorities)).to eql([cacert])
|
|
1163
|
-
end
|
|
1164
|
-
end
|
|
1165
|
-
|
|
1166
|
-
context "with java stores" do
|
|
1167
|
-
let(:do_register) { true }
|
|
1168
|
-
let(:keystore) { Stud::Temporary.file.path }
|
|
1169
|
-
let(:truststore) { Stud::Temporary.file.path }
|
|
1170
|
-
let(:options) { base_options.merge(
|
|
1171
|
-
"keystore" => keystore,
|
|
1172
|
-
"keystore_password" => "keystore",
|
|
1173
|
-
"truststore" => truststore,
|
|
1174
|
-
"truststore_password" => "truststore",
|
|
1175
|
-
"ssl_certificate_verification" => true
|
|
1176
|
-
) }
|
|
1177
|
-
|
|
1178
|
-
let(:spy_http_client_builder!) do
|
|
1179
|
-
allow(described_class::HttpClientBuilder).to receive(:build).with(any_args).and_call_original
|
|
1180
|
-
allow(described_class::HttpClientBuilder).to receive(:setup_ssl).with(any_args).and_return({})
|
|
1181
|
-
end
|
|
1182
|
-
|
|
1183
|
-
after :each do
|
|
1184
|
-
File.delete(keystore)
|
|
1185
|
-
File.delete(truststore)
|
|
1186
|
-
end
|
|
1187
|
-
|
|
1188
|
-
it "should map new configs into params" do
|
|
1189
|
-
expect(subject.params).to match hash_including(
|
|
1190
|
-
"ssl_enabled" => true,
|
|
1191
|
-
"ssl_keystore_path" => keystore,
|
|
1192
|
-
"ssl_truststore_path" => truststore,
|
|
1193
|
-
"ssl_verification_mode" => "full"
|
|
1194
|
-
)
|
|
1195
|
-
|
|
1196
|
-
expect(subject.params["ssl_keystore_password"].value).to eql("keystore")
|
|
1197
|
-
expect(subject.params["ssl_truststore_password"].value).to eql("truststore")
|
|
1198
|
-
end
|
|
1199
|
-
|
|
1200
|
-
it "should set new configs variables" do
|
|
1201
|
-
expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
|
|
1202
|
-
expect(subject.instance_variable_get(:@ssl_keystore_path)).to eql(keystore)
|
|
1203
|
-
expect(subject.instance_variable_get(:@ssl_keystore_password).value).to eql("keystore")
|
|
1204
|
-
expect(subject.instance_variable_get(:@ssl_truststore_path)).to eql(truststore)
|
|
1205
|
-
expect(subject.instance_variable_get(:@ssl_truststore_password).value).to eql("truststore")
|
|
1206
|
-
expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("full")
|
|
1207
|
-
end
|
|
1208
|
-
end
|
|
1209
|
-
end
|
|
1210
|
-
|
|
1211
1128
|
describe "retry_on_conflict" do
|
|
1212
1129
|
let(:num_retries) { 123 }
|
|
1213
1130
|
let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
|
|
@@ -1560,7 +1477,7 @@ describe LogStash::Outputs::ElasticSearch do
|
|
|
1560
1477
|
expect( event ).to be_a LogStash::Event
|
|
1561
1478
|
expect( event ).to be events.first
|
|
1562
1479
|
expect( reason ).to start_with "Could not index event to Elasticsearch. status: #{error_code}, action: [\"index\""
|
|
1563
|
-
expect( reason ).to match /_id
|
|
1480
|
+
expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
|
|
1564
1481
|
|
|
1565
1482
|
method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
|
|
1566
1483
|
end.once
|
|
@@ -1618,32 +1535,6 @@ describe LogStash::Outputs::ElasticSearch do
|
|
|
1618
1535
|
end; end
|
|
1619
1536
|
end
|
|
1620
1537
|
|
|
1621
|
-
context "with an already base64-encoded API key" do
|
|
1622
|
-
let(:api_key) { Base64.strict_encode64("some_id:some_api_key") }
|
|
1623
|
-
let(:options) { { "ssl_enabled" => true, "api_key" => ::LogStash::Util::Password.new(api_key) } }
|
|
1624
|
-
let(:base64_api_key) { "ApiKey #{api_key}" }
|
|
1625
|
-
|
|
1626
|
-
it_behaves_like 'secure api-key authenticated client'
|
|
1627
|
-
end
|
|
1628
|
-
|
|
1629
|
-
context "with an Elastic Cloud API key (essu_ prefix)" do
|
|
1630
|
-
let(:api_key) { "essu_VFZGblZreFhTekJ4ZDB4M2NHUnZRMEU2YzNWd1pYSnpaV055WlhRPQ==AAAAAAAA" }
|
|
1631
|
-
let(:options) { { "ssl_enabled" => true, "api_key" => ::LogStash::Util::Password.new(api_key) } }
|
|
1632
|
-
|
|
1633
|
-
it "sets the Authorization header verbatim without re-encoding" do
|
|
1634
|
-
expect(manticore_options[:headers]).to eq({ "Authorization" => "ApiKey #{api_key}" })
|
|
1635
|
-
end
|
|
1636
|
-
end
|
|
1637
|
-
|
|
1638
|
-
context "with an unrecognized api_key format" do
|
|
1639
|
-
let(:do_register) { false }
|
|
1640
|
-
let(:options) { { "ssl_enabled" => true, "api_key" => ::LogStash::Util::Password.new("not-a-valid-key") } }
|
|
1641
|
-
|
|
1642
|
-
it "fails registration with a configuration error" do
|
|
1643
|
-
expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Invalid api_key format/)
|
|
1644
|
-
end
|
|
1645
|
-
end
|
|
1646
|
-
|
|
1647
1538
|
context "when set without ssl_enabled => true" do
|
|
1648
1539
|
let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
|
|
1649
1540
|
let(:options) { { "api_key" => api_key } }
|
|
@@ -195,3 +195,25 @@ describe "SSL options" do
|
|
|
195
195
|
end
|
|
196
196
|
end
|
|
197
197
|
|
|
198
|
+
# Move outside the SSL options describe block that has the after hook
|
|
199
|
+
describe "SSL obsolete settings" do
|
|
200
|
+
let(:base_settings) { { "hosts" => "localhost", "pool_max" => 1, "pool_max_per_route" => 1 } }
|
|
201
|
+
[
|
|
202
|
+
{name: 'ssl', replacement: 'ssl_enabled'},
|
|
203
|
+
{name: 'ssl_certificate_verification', replacement: 'ssl_verification_mode'},
|
|
204
|
+
{name: 'cacert', replacement: 'ssl_certificate_authorities'},
|
|
205
|
+
{name: 'truststore', replacement: 'ssl_truststore_path'},
|
|
206
|
+
{name: 'keystore', replacement: 'ssl_keystore_path'},
|
|
207
|
+
{name: 'truststore_password', replacement: 'ssl_truststore_password'},
|
|
208
|
+
{name: 'keystore_password', replacement: 'ssl_keystore_password'}
|
|
209
|
+
].each do |obsolete_setting|
|
|
210
|
+
context "with option #{obsolete_setting[:name]}" do
|
|
211
|
+
let(:settings) { base_settings.merge(obsolete_setting[:name] => "value") }
|
|
212
|
+
|
|
213
|
+
it "emits an error about the setting being obsolete" do
|
|
214
|
+
error_text = /The setting `#{obsolete_setting[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. (Use|Set) '#{obsolete_setting[:replacement]}' instead/i
|
|
215
|
+
expect { LogStash::Outputs::ElasticSearch.new(settings) }.to raise_error LogStash::ConfigurationError, error_text
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
end
|
|
@@ -4,6 +4,7 @@ require_relative "../../../spec/es_spec_helper"
|
|
|
4
4
|
describe "whitelisting error types in expected behavior" do
|
|
5
5
|
let(:template) { '{"template" : "not important, will be updated by :index"}' }
|
|
6
6
|
let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
|
|
7
|
+
let(:action1) { ["index", {:_id=>1, :routing=>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1] }
|
|
7
8
|
let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } }
|
|
8
9
|
|
|
9
10
|
subject { LogStash::Outputs::ElasticSearch.new(settings) }
|