logstash-output-elasticsearch 11.22.13-java → 12.0.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -8
- data/docs/index.asciidoc +19 -106
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +4 -1
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +22 -11
- data/lib/logstash/outputs/elasticsearch/http_client.rb +29 -42
- data/lib/logstash/outputs/elasticsearch.rb +0 -47
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +12 -28
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/es_spec_helper.rb +22 -34
- data/spec/fixtures/test_certs/GENERATED_AT +1 -1
- data/spec/fixtures/test_certs/ca.crt +27 -30
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -1
- data/spec/fixtures/test_certs/renew.sh +3 -2
- data/spec/fixtures/test_certs/test.crt +28 -29
- data/spec/fixtures/test_certs/test.der.sha256 +1 -1
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +1 -3
- data/spec/integration/outputs/delete_spec.rb +4 -4
- data/spec/integration/outputs/ilm_spec.rb +12 -18
- data/spec/integration/outputs/index_spec.rb +2 -12
- data/spec/integration/outputs/index_version_spec.rb +7 -7
- data/spec/integration/outputs/painless_update_spec.rb +10 -11
- data/spec/integration/outputs/unsupported_actions_spec.rb +10 -15
- data/spec/integration/outputs/update_spec.rb +9 -11
- data/spec/spec_helper.rb +0 -8
- data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
- data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +0 -77
- data/spec/unit/outputs/elasticsearch_spec.rb +2 -82
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +22 -0
- data/spec/unit/outputs/error_whitelist_spec.rb +1 -0
- metadata +6 -2
@@ -270,83 +270,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
270
270
|
|
271
271
|
end
|
272
272
|
end
|
273
|
-
context "the 'user-agent' header" do
|
274
|
-
let(:pool) { double("pool") }
|
275
|
-
let(:compression_level) { 6 }
|
276
|
-
let(:base_options) { super().merge( :client_settings => {:compression_level => compression_level}) }
|
277
|
-
let(:actions) { [
|
278
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message_1"=> message_1}],
|
279
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message_2"=> message_2}],
|
280
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message_3"=> message_3}],
|
281
|
-
]}
|
282
|
-
let(:message_1) { "hello" }
|
283
|
-
let(:message_2_size) { 1_000 }
|
284
|
-
let(:message_2) { SecureRandom.alphanumeric(message_2_size / 2 ) * 2 }
|
285
|
-
let(:message_3_size) { 1_000 }
|
286
|
-
let(:message_3) { "m" * message_3_size }
|
287
|
-
let(:messages_size) { message_1.size + message_2.size + message_3.size }
|
288
|
-
let(:action_overhead) { 42 + 16 + 2 } # header plus doc key size plus new line overhead per action
|
289
|
-
|
290
|
-
let(:response) do
|
291
|
-
response = double("response")
|
292
|
-
allow(response).to receive(:code).and_return(response)
|
293
|
-
allow(response).to receive(:body).and_return({"errors" => false}.to_json)
|
294
|
-
response
|
295
|
-
end
|
296
|
-
|
297
|
-
before(:each) do
|
298
|
-
subject.instance_variable_set("@pool", pool)
|
299
|
-
end
|
300
|
-
|
301
|
-
it "carries bulk request's uncompressed size" do
|
302
|
-
expect(pool).to receive(:post) do |path, params, body|
|
303
|
-
headers = params.fetch(:headers, {})
|
304
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("3")
|
305
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
|
306
|
-
end.and_return(response)
|
307
|
-
|
308
|
-
subject.send(:bulk, actions)
|
309
|
-
end
|
310
|
-
context "without compression" do
|
311
|
-
let(:compression_level) { 0 }
|
312
|
-
it "carries bulk request's uncompressed size" do
|
313
|
-
expect(pool).to receive(:post) do |path, params, body|
|
314
|
-
headers = params.fetch(:headers, {})
|
315
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("3")
|
316
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
|
317
|
-
end.and_return(response)
|
318
|
-
subject.send(:bulk, actions)
|
319
|
-
end
|
320
|
-
end
|
321
|
-
|
322
|
-
context "with compressed messages over 20MB" do
|
323
|
-
let(:message_2_size) { 21_000_000 }
|
324
|
-
it "carries bulk request's uncompressed size" do
|
325
|
-
# only the first, tiny, message is sent first
|
326
|
-
expect(pool).to receive(:post) do |path, params, body|
|
327
|
-
headers = params.fetch(:headers, {})
|
328
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_1.size + action_overhead).to_s
|
329
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("1")
|
330
|
-
end.and_return(response)
|
331
|
-
|
332
|
-
# huge message_2 is sent afterwards alone
|
333
|
-
expect(pool).to receive(:post) do |path, params, body|
|
334
|
-
headers = params.fetch(:headers, {})
|
335
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_2.size + action_overhead).to_s
|
336
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("1")
|
337
|
-
end.and_return(response)
|
338
|
-
|
339
|
-
# finally medium message_3 is sent alone as well
|
340
|
-
expect(pool).to receive(:post) do |path, params, body|
|
341
|
-
headers = params.fetch(:headers, {})
|
342
|
-
expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_3.size + action_overhead).to_s
|
343
|
-
expect(headers["X-Elastic-Event-Count"]).to eq("1")
|
344
|
-
end.and_return(response)
|
345
|
-
|
346
|
-
subject.send(:bulk, actions)
|
347
|
-
end
|
348
|
-
end
|
349
|
-
end
|
350
273
|
end
|
351
274
|
|
352
275
|
describe "sniffing" do
|
@@ -777,7 +777,7 @@ describe LogStash::Outputs::ElasticSearch do
|
|
777
777
|
end
|
778
778
|
|
779
779
|
before(:each) do
|
780
|
-
allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)
|
780
|
+
allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
|
781
781
|
expect( stream.string ).to include '"foo":"bar1"'
|
782
782
|
expect( stream.string ).to include '"foo":"bar2"'
|
783
783
|
end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely
|
@@ -915,12 +915,7 @@ describe LogStash::Outputs::ElasticSearch do
|
|
915
915
|
allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body|
|
916
916
|
if body.length > max_bytes
|
917
917
|
max_bytes *= 2 # ensure a successful retry
|
918
|
-
|
919
|
-
413,
|
920
|
-
"test-url",
|
921
|
-
body,
|
922
|
-
""
|
923
|
-
)
|
918
|
+
double("Response", :code => 413, :body => "")
|
924
919
|
else
|
925
920
|
double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}')
|
926
921
|
end
|
@@ -1130,81 +1125,6 @@ describe LogStash::Outputs::ElasticSearch do
|
|
1130
1125
|
end
|
1131
1126
|
end
|
1132
1127
|
|
1133
|
-
describe "SSL deprecated settings" do
|
1134
|
-
let(:base_options) { {"ssl" => "true"} }
|
1135
|
-
|
1136
|
-
context "with client certificate" do
|
1137
|
-
let(:do_register) { true }
|
1138
|
-
let(:cacert) { Stud::Temporary.file.path }
|
1139
|
-
let(:options) { base_options.merge(
|
1140
|
-
"cacert" => cacert,
|
1141
|
-
"ssl_certificate_verification" => false
|
1142
|
-
) }
|
1143
|
-
|
1144
|
-
after :each do
|
1145
|
-
File.delete(cacert)
|
1146
|
-
end
|
1147
|
-
|
1148
|
-
it "should map new configs into params" do
|
1149
|
-
expect(subject.params).to match hash_including(
|
1150
|
-
"ssl_enabled" => true,
|
1151
|
-
"ssl_verification_mode" => "none",
|
1152
|
-
"ssl_certificate_authorities" => [cacert]
|
1153
|
-
)
|
1154
|
-
end
|
1155
|
-
|
1156
|
-
it "should set new configs variables" do
|
1157
|
-
expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
|
1158
|
-
expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("none")
|
1159
|
-
expect(subject.instance_variable_get(:@ssl_certificate_authorities)).to eql([cacert])
|
1160
|
-
end
|
1161
|
-
end
|
1162
|
-
|
1163
|
-
context "with java stores" do
|
1164
|
-
let(:do_register) { true }
|
1165
|
-
let(:keystore) { Stud::Temporary.file.path }
|
1166
|
-
let(:truststore) { Stud::Temporary.file.path }
|
1167
|
-
let(:options) { base_options.merge(
|
1168
|
-
"keystore" => keystore,
|
1169
|
-
"keystore_password" => "keystore",
|
1170
|
-
"truststore" => truststore,
|
1171
|
-
"truststore_password" => "truststore",
|
1172
|
-
"ssl_certificate_verification" => true
|
1173
|
-
) }
|
1174
|
-
|
1175
|
-
let(:spy_http_client_builder!) do
|
1176
|
-
allow(described_class::HttpClientBuilder).to receive(:build).with(any_args).and_call_original
|
1177
|
-
allow(described_class::HttpClientBuilder).to receive(:setup_ssl).with(any_args).and_return({})
|
1178
|
-
end
|
1179
|
-
|
1180
|
-
after :each do
|
1181
|
-
File.delete(keystore)
|
1182
|
-
File.delete(truststore)
|
1183
|
-
end
|
1184
|
-
|
1185
|
-
it "should map new configs into params" do
|
1186
|
-
expect(subject.params).to match hash_including(
|
1187
|
-
"ssl_enabled" => true,
|
1188
|
-
"ssl_keystore_path" => keystore,
|
1189
|
-
"ssl_truststore_path" => truststore,
|
1190
|
-
"ssl_verification_mode" => "full"
|
1191
|
-
)
|
1192
|
-
|
1193
|
-
expect(subject.params["ssl_keystore_password"].value).to eql("keystore")
|
1194
|
-
expect(subject.params["ssl_truststore_password"].value).to eql("truststore")
|
1195
|
-
end
|
1196
|
-
|
1197
|
-
it "should set new configs variables" do
|
1198
|
-
expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
|
1199
|
-
expect(subject.instance_variable_get(:@ssl_keystore_path)).to eql(keystore)
|
1200
|
-
expect(subject.instance_variable_get(:@ssl_keystore_password).value).to eql("keystore")
|
1201
|
-
expect(subject.instance_variable_get(:@ssl_truststore_path)).to eql(truststore)
|
1202
|
-
expect(subject.instance_variable_get(:@ssl_truststore_password).value).to eql("truststore")
|
1203
|
-
expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("full")
|
1204
|
-
end
|
1205
|
-
end
|
1206
|
-
end
|
1207
|
-
|
1208
1128
|
describe "retry_on_conflict" do
|
1209
1129
|
let(:num_retries) { 123 }
|
1210
1130
|
let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
|
@@ -195,3 +195,25 @@ describe "SSL options" do
|
|
195
195
|
end
|
196
196
|
end
|
197
197
|
|
198
|
+
# Move outside the SSL options describe block that has the after hook
|
199
|
+
describe "SSL obsolete settings" do
|
200
|
+
let(:base_settings) { { "hosts" => "localhost", "pool_max" => 1, "pool_max_per_route" => 1 } }
|
201
|
+
[
|
202
|
+
{name: 'ssl', replacement: 'ssl_enabled'},
|
203
|
+
{name: 'ssl_certificate_verification', replacement: 'ssl_verification_mode'},
|
204
|
+
{name: 'cacert', replacement: 'ssl_certificate_authorities'},
|
205
|
+
{name: 'truststore', replacement: 'ssl_truststore_path'},
|
206
|
+
{name: 'keystore', replacement: 'ssl_keystore_path'},
|
207
|
+
{name: 'truststore_password', replacement: 'ssl_truststore_password'},
|
208
|
+
{name: 'keystore_password', replacement: 'ssl_keystore_password'}
|
209
|
+
].each do |obsolete_setting|
|
210
|
+
context "with option #{obsolete_setting[:name]}" do
|
211
|
+
let(:settings) { base_settings.merge(obsolete_setting[:name] => "value") }
|
212
|
+
|
213
|
+
it "emits an error about the setting being obsolete" do
|
214
|
+
error_text = /The setting `#{obsolete_setting[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. (Use|Set) '#{obsolete_setting[:replacement]}' instead/i
|
215
|
+
expect { LogStash::Outputs::ElasticSearch.new(settings) }.to raise_error LogStash::ConfigurationError, error_text
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
@@ -4,6 +4,7 @@ require_relative "../../../spec/es_spec_helper"
|
|
4
4
|
describe "whitelisting error types in expected behavior" do
|
5
5
|
let(:template) { '{"template" : "not important, will be updated by :index"}' }
|
6
6
|
let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
|
7
|
+
let(:action1) { ["index", {:_id=>1, :routing=>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1] }
|
7
8
|
let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } }
|
8
9
|
|
9
10
|
subject { LogStash::Outputs::ElasticSearch.new(settings) }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 12.0.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-12-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -326,7 +326,9 @@ files:
|
|
326
326
|
- spec/integration/outputs/update_spec.rb
|
327
327
|
- spec/spec_helper.rb
|
328
328
|
- spec/support/elasticsearch/api/actions/delete_ilm_policy.rb
|
329
|
+
- spec/support/elasticsearch/api/actions/get_alias.rb
|
329
330
|
- spec/support/elasticsearch/api/actions/get_ilm_policy.rb
|
331
|
+
- spec/support/elasticsearch/api/actions/put_alias.rb
|
330
332
|
- spec/support/elasticsearch/api/actions/put_ilm_policy.rb
|
331
333
|
- spec/unit/http_client_builder_spec.rb
|
332
334
|
- spec/unit/outputs/elasticsearch/data_stream_support_spec.rb
|
@@ -413,7 +415,9 @@ test_files:
|
|
413
415
|
- spec/integration/outputs/update_spec.rb
|
414
416
|
- spec/spec_helper.rb
|
415
417
|
- spec/support/elasticsearch/api/actions/delete_ilm_policy.rb
|
418
|
+
- spec/support/elasticsearch/api/actions/get_alias.rb
|
416
419
|
- spec/support/elasticsearch/api/actions/get_ilm_policy.rb
|
420
|
+
- spec/support/elasticsearch/api/actions/put_alias.rb
|
417
421
|
- spec/support/elasticsearch/api/actions/put_ilm_policy.rb
|
418
422
|
- spec/unit/http_client_builder_spec.rb
|
419
423
|
- spec/unit/outputs/elasticsearch/data_stream_support_spec.rb
|