logstash-output-elasticsearch 11.12.4-java → 11.15.9-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -289,8 +289,8 @@ describe "indexing" do
289
289
  "hosts" => [ get_host_port ],
290
290
  "user" => user,
291
291
  "password" => password,
292
- "ssl" => true,
293
- "cacert" => cacert,
292
+ "ssl_enabled" => true,
293
+ "ssl_certificate_authorities" => cacert,
294
294
  "index" => index
295
295
  }
296
296
  end
@@ -302,7 +302,7 @@ describe "indexing" do
302
302
 
303
303
  context "when no keystore nor ca cert set and verification is disabled" do
304
304
  let(:config) do
305
- super().tap { |config| config.delete('cacert') }.merge('ssl_certificate_verification' => false)
305
+ super().tap { |config| config.delete('ssl_certificate_authorities') }.merge('ssl_verification_mode' => 'none')
306
306
  end
307
307
 
308
308
  include_examples("an indexer", true)
@@ -311,9 +311,9 @@ describe "indexing" do
311
311
  context "when keystore is set and verification is disabled" do
312
312
  let(:config) do
313
313
  super().merge(
314
- 'ssl_certificate_verification' => false,
315
- 'keystore' => 'spec/fixtures/test_certs/test.p12',
316
- 'keystore_password' => '1234567890'
314
+ 'ssl_verification_mode' => 'none',
315
+ 'ssl_keystore_path' => 'spec/fixtures/test_certs/test.p12',
316
+ 'ssl_keystore_password' => '1234567890'
317
317
  )
318
318
  end
319
319
 
@@ -322,10 +322,10 @@ describe "indexing" do
322
322
 
323
323
  context "when keystore has self-signed cert and verification is disabled" do
324
324
  let(:config) do
325
- super().tap { |config| config.delete('cacert') }.merge(
326
- 'ssl_certificate_verification' => false,
327
- 'keystore' => 'spec/fixtures/test_certs/test_self_signed.p12',
328
- 'keystore_password' => '1234567890'
325
+ super().tap { |config| config.delete('ssl_certificate_authorities') }.merge(
326
+ 'ssl_verification_mode' => 'none',
327
+ 'ssl_keystore_path' => 'spec/fixtures/test_certs/test_self_signed.p12',
328
+ 'ssl_keystore_password' => '1234567890'
329
329
  )
330
330
  end
331
331
 
@@ -349,8 +349,8 @@ describe "indexing" do
349
349
  let(:config) do
350
350
  {
351
351
  "hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
352
- "ssl" => true,
353
- "cacert" => "spec/fixtures/test_certs/test.crt",
352
+ "ssl_enabled" => true,
353
+ "ssl_certificate_authorities" => "spec/fixtures/test_certs/test.crt",
354
354
  "index" => index
355
355
  }
356
356
  end
@@ -358,10 +358,10 @@ describe "indexing" do
358
358
  include_examples("an indexer", true)
359
359
  end
360
360
 
361
- context "without providing `cacert`" do
361
+ context "without providing `ssl_certificate_authorities`" do
362
362
  let(:config) do
363
363
  super().tap do |c|
364
- c.delete("cacert")
364
+ c.delete("ssl_certificate_authorities")
365
365
  end
366
366
  end
367
367
 
@@ -369,10 +369,10 @@ describe "indexing" do
369
369
  end
370
370
 
371
371
  if Gem::Version.new(LOGSTASH_VERSION) >= Gem::Version.new("8.3.0")
372
- context "with `ca_trusted_fingerprint` instead of `cacert`" do
372
+ context "with `ca_trusted_fingerprint` instead of `ssl_certificate_authorities`" do
373
373
  let(:config) do
374
374
  super().tap do |c|
375
- c.delete("cacert")
375
+ c.delete("ssl_certificate_authorities")
376
376
  c.update("ca_trusted_fingerprint" => ca_trusted_fingerprint)
377
377
  end
378
378
  end
@@ -23,7 +23,9 @@ describe "index template expected behavior", :integration => true do
23
23
 
24
24
  elasticsearch_client.indices.delete_template(:name => '*')
25
25
  # This can fail if there are no indexes, ignore failure.
26
- elasticsearch_client.indices.delete(:index => '*') rescue nil
26
+ elasticsearch_client.indices.delete(:index => '*') rescue puts("DELETE INDICES ERROR: #{$!}")
27
+ # Since we are pinned to ES client 7.x, we need to delete data streams the hard way...
28
+ elasticsearch_client.perform_request("DELETE", "/_data_stream/*") rescue puts("DELETE DATA STREAMS ERROR: #{$!}")
27
29
  end
28
30
 
29
31
  context 'with ecs_compatibility => disabled' do
@@ -48,19 +50,19 @@ describe "index template expected behavior", :integration => true do
48
50
 
49
51
  # Wait or fail until everything's indexed.
50
52
  Stud::try(20.times) do
51
- r = @es.search(index: 'logstash-*')
53
+ r = @es.search(index: 'logstash*')
52
54
  expect(r).to have_hits(8)
53
55
  end
54
56
  end
55
57
 
56
58
  it "permits phrase searching on string fields" do
57
- results = @es.search(:q => "message:\"sample message\"")
59
+ results = @es.search(index: 'logstash*', q: "message:\"sample message\"")
58
60
  expect(results).to have_hits(1)
59
61
  expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
60
62
  end
61
63
 
62
64
  it "numbers dynamically map to a numeric type and permit range queries" do
63
- results = @es.search(:q => "somevalue:[5 TO 105]")
65
+ results = @es.search(index: 'logstash*', q: "somevalue:[5 TO 105]")
64
66
  expect(results).to have_hits(2)
65
67
 
66
68
  values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
@@ -70,22 +72,22 @@ describe "index template expected behavior", :integration => true do
70
72
  end
71
73
 
72
74
  it "does not create .keyword field for top-level message field" do
73
- results = @es.search(:q => "message.keyword:\"sample message here\"")
75
+ results = @es.search(index: 'logstash*', q: "message.keyword:\"sample message here\"")
74
76
  expect(results).to have_hits(0)
75
77
  end
76
78
 
77
79
  it "creates .keyword field for nested message fields" do
78
- results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
80
+ results = @es.search(index: 'logstash*', q: "somemessage.message.keyword:\"sample nested message here\"")
79
81
  expect(results).to have_hits(1)
80
82
  end
81
83
 
82
84
  it "creates .keyword field from any string field which is not_analyzed" do
83
- results = @es.search(:q => "country.keyword:\"us\"")
85
+ results = @es.search(index: 'logstash*', q: "country.keyword:\"us\"")
84
86
  expect(results).to have_hits(1)
85
87
  expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
86
88
 
87
89
  # partial or terms should not work.
88
- results = @es.search(:q => "country.keyword:\"u\"")
90
+ results = @es.search(index: 'logstash*', q: "country.keyword:\"u\"")
89
91
  expect(results).to have_hits(0)
90
92
  end
91
93
 
@@ -94,7 +96,7 @@ describe "index template expected behavior", :integration => true do
94
96
  end
95
97
 
96
98
  it "aggregate .keyword results correctly " do
97
- results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
99
+ results = @es.search(index: 'logstash*', body: { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
98
100
  terms = results["buckets"].collect { |b| b["key"] }
99
101
 
100
102
  expect(terms).to include("us")
@@ -114,7 +114,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
114
114
  {
115
115
  'hosts' => [ 'http://127.0.0.1:12345' ],
116
116
  'http_compression' => 'true', 'bulk_path' => '_bulk', 'timeout' => '30',
117
- 'user' => 'elastic', 'password' => 'ForSearch!', 'ssl' => 'false'
117
+ 'user' => 'elastic', 'password' => 'ForSearch!', 'ssl_enabled' => 'false'
118
118
  }
119
119
  end
120
120
 
@@ -164,7 +164,9 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
164
164
  'data_stream_dataset' => 'any',
165
165
  'data_stream_namespace' => 'any',
166
166
  'data_stream_sync_fields' => true,
167
- 'data_stream_auto_routing' => true)
167
+ 'data_stream_auto_routing' => true,
168
+ 'dlq_custom_codes' => [ 404 ],
169
+ 'dlq_on_failed_indexname_interpolation' => false)
168
170
  }
169
171
 
170
172
  it 'should enable data-streams by default' do
@@ -50,6 +50,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
50
50
 
51
51
  describe "healthcheck url handling" do
52
52
  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
53
+ let(:success_response) { double("Response", :code => 200) }
54
+
53
55
  before(:example) do
54
56
  expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
55
57
  expect(url.path).to be_empty
@@ -60,6 +62,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
60
62
  it "performs the healthcheck to the root" do
61
63
  expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
62
64
  expect(url.path).to be_empty
65
+
66
+ success_response
63
67
  end
64
68
  expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
65
69
  end
@@ -71,6 +75,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
71
75
  it "performs the healthcheck to the healthcheck_path" do
72
76
  expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
73
77
  expect(url.path).to be_empty
78
+
79
+ success_response
74
80
  end
75
81
  expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
76
82
  end
@@ -197,9 +203,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
197
203
  "build_flavor" => 'default'}
198
204
  })
199
205
  end
206
+ let(:success_response) { double("head_req", :code => 200)}
200
207
 
201
208
  before :each do
202
- allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
209
+ allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil).and_return(success_response)
203
210
  allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
204
211
  end
205
212
  let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
@@ -225,7 +232,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
225
232
  u,m = subject.get_connection
226
233
 
227
234
  # The resurrectionist will call this to check on the backend
228
- response = double("response")
235
+ response = double("response", :code => 200)
229
236
  expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
230
237
 
231
238
  subject.return_connection(u)
@@ -1,4 +1,4 @@
1
- require "logstash/devutils/rspec/spec_helper"
1
+ require_relative "../../../../spec/spec_helper"
2
2
  require "logstash/outputs/elasticsearch/template_manager"
3
3
 
4
4
  describe LogStash::Outputs::ElasticSearch::TemplateManager do
@@ -33,33 +33,85 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
33
33
  end
34
34
  end
35
35
 
36
- describe "index template with ilm settings" do
36
+ context "index template with ilm settings" do
37
37
  let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
38
38
  let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
39
39
 
40
- describe "in version 8+" do
41
- let(:file_path) { described_class.default_template_path(8) }
42
- let(:template) { described_class.read_template_file(file_path)}
40
+ describe "with custom template" do
43
41
 
44
- it "should update settings" do
45
- expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
46
- described_class.add_ilm_settings_to_template(plugin, template)
47
- expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
48
- expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
49
- expect(template.include?('settings')).to be_falsey
42
+ describe "in version 8+" do
43
+ let(:file_path) { described_class.default_template_path(8) }
44
+ let(:template) { described_class.read_template_file(file_path)}
45
+
46
+ it "should update settings" do
47
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
48
+ described_class.add_ilm_settings_to_template(plugin, template)
49
+ expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
50
+ expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
51
+ expect(template.include?('settings')).to be_falsey
52
+ end
53
+ end
54
+
55
+ describe "in version < 8" do
56
+ let(:file_path) { described_class.default_template_path(7) }
57
+ let(:template) { described_class.read_template_file(file_path)}
58
+
59
+ it "should update settings" do
60
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
61
+ described_class.add_ilm_settings_to_template(plugin, template)
62
+ expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
63
+ expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
64
+ expect(template.include?('template')).to be_falsey
65
+ end
50
66
  end
51
67
  end
52
68
 
53
- describe "in version < 8" do
54
- let(:file_path) { described_class.default_template_path(7) }
55
- let(:template) { described_class.read_template_file(file_path)}
69
+ context "resolve template setting" do
70
+ let(:plugin_settings) { super().merge({"template_api" => template_api}) }
71
+
72
+ describe "with composable template API" do
73
+ let(:template_api) { "composable" }
56
74
 
57
- it "should update settings" do
58
- expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
59
- described_class.add_ilm_settings_to_template(plugin, template)
60
- expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
61
- expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
62
- expect(template.include?('template')).to be_falsey
75
+ it 'resolves composable index template API compatible setting' do
76
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log
77
+ template = {}
78
+ described_class.resolve_template_settings(plugin, template)
79
+ expect(template["template"]["settings"]).not_to eq(nil)
80
+ end
81
+ end
82
+
83
+ describe "with legacy template API" do
84
+ let(:template_api) { "legacy" }
85
+
86
+ it 'resolves legacy index template API compatible setting' do
87
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log
88
+ template = {}
89
+ described_class.resolve_template_settings(plugin, template)
90
+ expect(template["settings"]).not_to eq(nil)
91
+ end
92
+ end
93
+
94
+ describe "with `template_api => 'auto'`" do
95
+ let(:template_api) { "auto" }
96
+
97
+ describe "with ES < 8 versions" do
98
+
99
+ it 'resolves legacy index template API compatible setting' do
100
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
101
+ template = {}
102
+ described_class.resolve_template_settings(plugin, template)
103
+ expect(template["settings"]).not_to eq(nil)
104
+ end
105
+ end
106
+
107
+ describe "with ES >= 8 versions" do
108
+ it 'resolves composable index template API compatible setting' do
109
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
110
+ template = {}
111
+ described_class.resolve_template_settings(plugin, template)
112
+ expect(template["template"]["settings"]).not_to eq(nil)
113
+ end
114
+ end
63
115
  end
64
116
  end
65
117
  end