logstash-output-elasticsearch 11.22.13-java → 12.0.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +11 -8
  3. data/docs/index.asciidoc +19 -106
  4. data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +4 -1
  5. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +22 -11
  6. data/lib/logstash/outputs/elasticsearch/http_client.rb +29 -42
  7. data/lib/logstash/outputs/elasticsearch.rb +0 -47
  8. data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +12 -28
  9. data/logstash-output-elasticsearch.gemspec +1 -1
  10. data/spec/es_spec_helper.rb +22 -34
  11. data/spec/fixtures/test_certs/GENERATED_AT +1 -1
  12. data/spec/fixtures/test_certs/ca.crt +27 -30
  13. data/spec/fixtures/test_certs/ca.der.sha256 +1 -1
  14. data/spec/fixtures/test_certs/renew.sh +3 -2
  15. data/spec/fixtures/test_certs/test.crt +28 -29
  16. data/spec/fixtures/test_certs/test.der.sha256 +1 -1
  17. data/spec/fixtures/test_certs/test.p12 +0 -0
  18. data/spec/integration/outputs/compressed_indexing_spec.rb +1 -3
  19. data/spec/integration/outputs/delete_spec.rb +4 -4
  20. data/spec/integration/outputs/ilm_spec.rb +12 -18
  21. data/spec/integration/outputs/index_spec.rb +2 -12
  22. data/spec/integration/outputs/index_version_spec.rb +7 -7
  23. data/spec/integration/outputs/painless_update_spec.rb +10 -11
  24. data/spec/integration/outputs/unsupported_actions_spec.rb +10 -15
  25. data/spec/integration/outputs/update_spec.rb +9 -11
  26. data/spec/spec_helper.rb +0 -8
  27. data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
  28. data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
  29. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +0 -77
  30. data/spec/unit/outputs/elasticsearch_spec.rb +2 -82
  31. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +22 -0
  32. data/spec/unit/outputs/error_whitelist_spec.rb +1 -0
  33. metadata +6 -2
@@ -270,83 +270,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
270
270
 
271
271
  end
272
272
  end
273
- context "the 'user-agent' header" do
274
- let(:pool) { double("pool") }
275
- let(:compression_level) { 6 }
276
- let(:base_options) { super().merge( :client_settings => {:compression_level => compression_level}) }
277
- let(:actions) { [
278
- ["index", {:_id=>nil, :_index=>"logstash"}, {"message_1"=> message_1}],
279
- ["index", {:_id=>nil, :_index=>"logstash"}, {"message_2"=> message_2}],
280
- ["index", {:_id=>nil, :_index=>"logstash"}, {"message_3"=> message_3}],
281
- ]}
282
- let(:message_1) { "hello" }
283
- let(:message_2_size) { 1_000 }
284
- let(:message_2) { SecureRandom.alphanumeric(message_2_size / 2 ) * 2 }
285
- let(:message_3_size) { 1_000 }
286
- let(:message_3) { "m" * message_3_size }
287
- let(:messages_size) { message_1.size + message_2.size + message_3.size }
288
- let(:action_overhead) { 42 + 16 + 2 } # header plus doc key size plus new line overhead per action
289
-
290
- let(:response) do
291
- response = double("response")
292
- allow(response).to receive(:code).and_return(response)
293
- allow(response).to receive(:body).and_return({"errors" => false}.to_json)
294
- response
295
- end
296
-
297
- before(:each) do
298
- subject.instance_variable_set("@pool", pool)
299
- end
300
-
301
- it "carries bulk request's uncompressed size" do
302
- expect(pool).to receive(:post) do |path, params, body|
303
- headers = params.fetch(:headers, {})
304
- expect(headers["X-Elastic-Event-Count"]).to eq("3")
305
- expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
306
- end.and_return(response)
307
-
308
- subject.send(:bulk, actions)
309
- end
310
- context "without compression" do
311
- let(:compression_level) { 0 }
312
- it "carries bulk request's uncompressed size" do
313
- expect(pool).to receive(:post) do |path, params, body|
314
- headers = params.fetch(:headers, {})
315
- expect(headers["X-Elastic-Event-Count"]).to eq("3")
316
- expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
317
- end.and_return(response)
318
- subject.send(:bulk, actions)
319
- end
320
- end
321
-
322
- context "with compressed messages over 20MB" do
323
- let(:message_2_size) { 21_000_000 }
324
- it "carries bulk request's uncompressed size" do
325
- # only the first, tiny, message is sent first
326
- expect(pool).to receive(:post) do |path, params, body|
327
- headers = params.fetch(:headers, {})
328
- expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_1.size + action_overhead).to_s
329
- expect(headers["X-Elastic-Event-Count"]).to eq("1")
330
- end.and_return(response)
331
-
332
- # huge message_2 is sent afterwards alone
333
- expect(pool).to receive(:post) do |path, params, body|
334
- headers = params.fetch(:headers, {})
335
- expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_2.size + action_overhead).to_s
336
- expect(headers["X-Elastic-Event-Count"]).to eq("1")
337
- end.and_return(response)
338
-
339
- # finally medium message_3 is sent alone as well
340
- expect(pool).to receive(:post) do |path, params, body|
341
- headers = params.fetch(:headers, {})
342
- expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_3.size + action_overhead).to_s
343
- expect(headers["X-Elastic-Event-Count"]).to eq("1")
344
- end.and_return(response)
345
-
346
- subject.send(:bulk, actions)
347
- end
348
- end
349
- end
350
273
  end
351
274
 
352
275
  describe "sniffing" do
@@ -777,7 +777,7 @@ describe LogStash::Outputs::ElasticSearch do
777
777
  end
778
778
 
779
779
  before(:each) do
780
- allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array), instance_of(Hash)) do |stream, actions, headers|
780
+ allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
781
781
  expect( stream.string ).to include '"foo":"bar1"'
782
782
  expect( stream.string ).to include '"foo":"bar2"'
783
783
  end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely
@@ -915,12 +915,7 @@ describe LogStash::Outputs::ElasticSearch do
915
915
  allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body|
916
916
  if body.length > max_bytes
917
917
  max_bytes *= 2 # ensure a successful retry
918
- raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
919
- 413,
920
- "test-url",
921
- body,
922
- ""
923
- )
918
+ double("Response", :code => 413, :body => "")
924
919
  else
925
920
  double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}')
926
921
  end
@@ -1130,81 +1125,6 @@ describe LogStash::Outputs::ElasticSearch do
1130
1125
  end
1131
1126
  end
1132
1127
 
1133
- describe "SSL deprecated settings" do
1134
- let(:base_options) { {"ssl" => "true"} }
1135
-
1136
- context "with client certificate" do
1137
- let(:do_register) { true }
1138
- let(:cacert) { Stud::Temporary.file.path }
1139
- let(:options) { base_options.merge(
1140
- "cacert" => cacert,
1141
- "ssl_certificate_verification" => false
1142
- ) }
1143
-
1144
- after :each do
1145
- File.delete(cacert)
1146
- end
1147
-
1148
- it "should map new configs into params" do
1149
- expect(subject.params).to match hash_including(
1150
- "ssl_enabled" => true,
1151
- "ssl_verification_mode" => "none",
1152
- "ssl_certificate_authorities" => [cacert]
1153
- )
1154
- end
1155
-
1156
- it "should set new configs variables" do
1157
- expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
1158
- expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("none")
1159
- expect(subject.instance_variable_get(:@ssl_certificate_authorities)).to eql([cacert])
1160
- end
1161
- end
1162
-
1163
- context "with java stores" do
1164
- let(:do_register) { true }
1165
- let(:keystore) { Stud::Temporary.file.path }
1166
- let(:truststore) { Stud::Temporary.file.path }
1167
- let(:options) { base_options.merge(
1168
- "keystore" => keystore,
1169
- "keystore_password" => "keystore",
1170
- "truststore" => truststore,
1171
- "truststore_password" => "truststore",
1172
- "ssl_certificate_verification" => true
1173
- ) }
1174
-
1175
- let(:spy_http_client_builder!) do
1176
- allow(described_class::HttpClientBuilder).to receive(:build).with(any_args).and_call_original
1177
- allow(described_class::HttpClientBuilder).to receive(:setup_ssl).with(any_args).and_return({})
1178
- end
1179
-
1180
- after :each do
1181
- File.delete(keystore)
1182
- File.delete(truststore)
1183
- end
1184
-
1185
- it "should map new configs into params" do
1186
- expect(subject.params).to match hash_including(
1187
- "ssl_enabled" => true,
1188
- "ssl_keystore_path" => keystore,
1189
- "ssl_truststore_path" => truststore,
1190
- "ssl_verification_mode" => "full"
1191
- )
1192
-
1193
- expect(subject.params["ssl_keystore_password"].value).to eql("keystore")
1194
- expect(subject.params["ssl_truststore_password"].value).to eql("truststore")
1195
- end
1196
-
1197
- it "should set new configs variables" do
1198
- expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
1199
- expect(subject.instance_variable_get(:@ssl_keystore_path)).to eql(keystore)
1200
- expect(subject.instance_variable_get(:@ssl_keystore_password).value).to eql("keystore")
1201
- expect(subject.instance_variable_get(:@ssl_truststore_path)).to eql(truststore)
1202
- expect(subject.instance_variable_get(:@ssl_truststore_password).value).to eql("truststore")
1203
- expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("full")
1204
- end
1205
- end
1206
- end
1207
-
1208
1128
  describe "retry_on_conflict" do
1209
1129
  let(:num_retries) { 123 }
1210
1130
  let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
@@ -195,3 +195,25 @@ describe "SSL options" do
195
195
  end
196
196
  end
197
197
 
198
+ # Move outside the SSL options describe block that has the after hook
199
+ describe "SSL obsolete settings" do
200
+ let(:base_settings) { { "hosts" => "localhost", "pool_max" => 1, "pool_max_per_route" => 1 } }
201
+ [
202
+ {name: 'ssl', replacement: 'ssl_enabled'},
203
+ {name: 'ssl_certificate_verification', replacement: 'ssl_verification_mode'},
204
+ {name: 'cacert', replacement: 'ssl_certificate_authorities'},
205
+ {name: 'truststore', replacement: 'ssl_truststore_path'},
206
+ {name: 'keystore', replacement: 'ssl_keystore_path'},
207
+ {name: 'truststore_password', replacement: 'ssl_truststore_password'},
208
+ {name: 'keystore_password', replacement: 'ssl_keystore_password'}
209
+ ].each do |obsolete_setting|
210
+ context "with option #{obsolete_setting[:name]}" do
211
+ let(:settings) { base_settings.merge(obsolete_setting[:name] => "value") }
212
+
213
+ it "emits an error about the setting being obsolete" do
214
+ error_text = /The setting `#{obsolete_setting[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. (Use|Set) '#{obsolete_setting[:replacement]}' instead/i
215
+ expect { LogStash::Outputs::ElasticSearch.new(settings) }.to raise_error LogStash::ConfigurationError, error_text
216
+ end
217
+ end
218
+ end
219
+ end
@@ -4,6 +4,7 @@ require_relative "../../../spec/es_spec_helper"
4
4
  describe "whitelisting error types in expected behavior" do
5
5
  let(:template) { '{"template" : "not important, will be updated by :index"}' }
6
6
  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
7
+ let(:action1) { ["index", {:_id=>1, :routing=>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1] }
7
8
  let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } }
8
9
 
9
10
  subject { LogStash::Outputs::ElasticSearch.new(settings) }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.22.13
4
+ version: 12.0.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2025-08-01 00:00:00.000000000 Z
11
+ date: 2024-12-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -326,7 +326,9 @@ files:
326
326
  - spec/integration/outputs/update_spec.rb
327
327
  - spec/spec_helper.rb
328
328
  - spec/support/elasticsearch/api/actions/delete_ilm_policy.rb
329
+ - spec/support/elasticsearch/api/actions/get_alias.rb
329
330
  - spec/support/elasticsearch/api/actions/get_ilm_policy.rb
331
+ - spec/support/elasticsearch/api/actions/put_alias.rb
330
332
  - spec/support/elasticsearch/api/actions/put_ilm_policy.rb
331
333
  - spec/unit/http_client_builder_spec.rb
332
334
  - spec/unit/outputs/elasticsearch/data_stream_support_spec.rb
@@ -413,7 +415,9 @@ test_files:
413
415
  - spec/integration/outputs/update_spec.rb
414
416
  - spec/spec_helper.rb
415
417
  - spec/support/elasticsearch/api/actions/delete_ilm_policy.rb
418
+ - spec/support/elasticsearch/api/actions/get_alias.rb
416
419
  - spec/support/elasticsearch/api/actions/get_ilm_policy.rb
420
+ - spec/support/elasticsearch/api/actions/put_alias.rb
417
421
  - spec/support/elasticsearch/api/actions/put_ilm_policy.rb
418
422
  - spec/unit/http_client_builder_spec.rb
419
423
  - spec/unit/outputs/elasticsearch/data_stream_support_spec.rb