logstash-output-elasticsearch 11.12.4-java → 11.15.9-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +58 -0
- data/docs/index.asciidoc +214 -66
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +14 -4
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +46 -19
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +26 -3
- data/lib/logstash/outputs/elasticsearch.rb +89 -39
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +60 -8
- data/lib/logstash/plugin_mixins/elasticsearch/common.rb +42 -18
- data/logstash-output-elasticsearch.gemspec +3 -2
- data/spec/es_spec_helper.rb +12 -7
- data/spec/fixtures/test_certs/ca.crt +13 -13
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -1
- data/spec/fixtures/test_certs/test.crt +14 -14
- data/spec/fixtures/test_certs/test.der.sha256 +1 -1
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/integration/outputs/index_spec.rb +16 -16
- data/spec/integration/outputs/templates_spec.rb +11 -9
- data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +4 -2
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +9 -2
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +72 -20
- data/spec/unit/outputs/elasticsearch_spec.rb +358 -28
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +166 -50
- metadata +18 -4
@@ -289,8 +289,8 @@ describe "indexing" do
|
|
289
289
|
"hosts" => [ get_host_port ],
|
290
290
|
"user" => user,
|
291
291
|
"password" => password,
|
292
|
-
"
|
293
|
-
"
|
292
|
+
"ssl_enabled" => true,
|
293
|
+
"ssl_certificate_authorities" => cacert,
|
294
294
|
"index" => index
|
295
295
|
}
|
296
296
|
end
|
@@ -302,7 +302,7 @@ describe "indexing" do
|
|
302
302
|
|
303
303
|
context "when no keystore nor ca cert set and verification is disabled" do
|
304
304
|
let(:config) do
|
305
|
-
super().tap { |config| config.delete('
|
305
|
+
super().tap { |config| config.delete('ssl_certificate_authorities') }.merge('ssl_verification_mode' => 'none')
|
306
306
|
end
|
307
307
|
|
308
308
|
include_examples("an indexer", true)
|
@@ -311,9 +311,9 @@ describe "indexing" do
|
|
311
311
|
context "when keystore is set and verification is disabled" do
|
312
312
|
let(:config) do
|
313
313
|
super().merge(
|
314
|
-
'
|
315
|
-
'
|
316
|
-
'
|
314
|
+
'ssl_verification_mode' => 'none',
|
315
|
+
'ssl_keystore_path' => 'spec/fixtures/test_certs/test.p12',
|
316
|
+
'ssl_keystore_password' => '1234567890'
|
317
317
|
)
|
318
318
|
end
|
319
319
|
|
@@ -322,10 +322,10 @@ describe "indexing" do
|
|
322
322
|
|
323
323
|
context "when keystore has self-signed cert and verification is disabled" do
|
324
324
|
let(:config) do
|
325
|
-
super().tap { |config| config.delete('
|
326
|
-
'
|
327
|
-
'
|
328
|
-
'
|
325
|
+
super().tap { |config| config.delete('ssl_certificate_authorities') }.merge(
|
326
|
+
'ssl_verification_mode' => 'none',
|
327
|
+
'ssl_keystore_path' => 'spec/fixtures/test_certs/test_self_signed.p12',
|
328
|
+
'ssl_keystore_password' => '1234567890'
|
329
329
|
)
|
330
330
|
end
|
331
331
|
|
@@ -349,8 +349,8 @@ describe "indexing" do
|
|
349
349
|
let(:config) do
|
350
350
|
{
|
351
351
|
"hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
|
352
|
-
"
|
353
|
-
"
|
352
|
+
"ssl_enabled" => true,
|
353
|
+
"ssl_certificate_authorities" => "spec/fixtures/test_certs/test.crt",
|
354
354
|
"index" => index
|
355
355
|
}
|
356
356
|
end
|
@@ -358,10 +358,10 @@ describe "indexing" do
|
|
358
358
|
include_examples("an indexer", true)
|
359
359
|
end
|
360
360
|
|
361
|
-
context "without providing `
|
361
|
+
context "without providing `ssl_certificate_authorities`" do
|
362
362
|
let(:config) do
|
363
363
|
super().tap do |c|
|
364
|
-
c.delete("
|
364
|
+
c.delete("ssl_certificate_authorities")
|
365
365
|
end
|
366
366
|
end
|
367
367
|
|
@@ -369,10 +369,10 @@ describe "indexing" do
|
|
369
369
|
end
|
370
370
|
|
371
371
|
if Gem::Version.new(LOGSTASH_VERSION) >= Gem::Version.new("8.3.0")
|
372
|
-
context "with `ca_trusted_fingerprint` instead of `
|
372
|
+
context "with `ca_trusted_fingerprint` instead of `ssl_certificate_authorities`" do
|
373
373
|
let(:config) do
|
374
374
|
super().tap do |c|
|
375
|
-
c.delete("
|
375
|
+
c.delete("ssl_certificate_authorities")
|
376
376
|
c.update("ca_trusted_fingerprint" => ca_trusted_fingerprint)
|
377
377
|
end
|
378
378
|
end
|
@@ -23,7 +23,9 @@ describe "index template expected behavior", :integration => true do
|
|
23
23
|
|
24
24
|
elasticsearch_client.indices.delete_template(:name => '*')
|
25
25
|
# This can fail if there are no indexes, ignore failure.
|
26
|
-
elasticsearch_client.indices.delete(:index => '*') rescue
|
26
|
+
elasticsearch_client.indices.delete(:index => '*') rescue puts("DELETE INDICES ERROR: #{$!}")
|
27
|
+
# Since we are pinned to ES client 7.x, we need to delete data streams the hard way...
|
28
|
+
elasticsearch_client.perform_request("DELETE", "/_data_stream/*") rescue puts("DELETE DATA STREAMS ERROR: #{$!}")
|
27
29
|
end
|
28
30
|
|
29
31
|
context 'with ecs_compatibility => disabled' do
|
@@ -48,19 +50,19 @@ describe "index template expected behavior", :integration => true do
|
|
48
50
|
|
49
51
|
# Wait or fail until everything's indexed.
|
50
52
|
Stud::try(20.times) do
|
51
|
-
r = @es.search(index: 'logstash
|
53
|
+
r = @es.search(index: 'logstash*')
|
52
54
|
expect(r).to have_hits(8)
|
53
55
|
end
|
54
56
|
end
|
55
57
|
|
56
58
|
it "permits phrase searching on string fields" do
|
57
|
-
results = @es.search(:q
|
59
|
+
results = @es.search(index: 'logstash*', q: "message:\"sample message\"")
|
58
60
|
expect(results).to have_hits(1)
|
59
61
|
expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
|
60
62
|
end
|
61
63
|
|
62
64
|
it "numbers dynamically map to a numeric type and permit range queries" do
|
63
|
-
results = @es.search(:q
|
65
|
+
results = @es.search(index: 'logstash*', q: "somevalue:[5 TO 105]")
|
64
66
|
expect(results).to have_hits(2)
|
65
67
|
|
66
68
|
values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
|
@@ -70,22 +72,22 @@ describe "index template expected behavior", :integration => true do
|
|
70
72
|
end
|
71
73
|
|
72
74
|
it "does not create .keyword field for top-level message field" do
|
73
|
-
results = @es.search(:q
|
75
|
+
results = @es.search(index: 'logstash*', q: "message.keyword:\"sample message here\"")
|
74
76
|
expect(results).to have_hits(0)
|
75
77
|
end
|
76
78
|
|
77
79
|
it "creates .keyword field for nested message fields" do
|
78
|
-
results = @es.search(:q
|
80
|
+
results = @es.search(index: 'logstash*', q: "somemessage.message.keyword:\"sample nested message here\"")
|
79
81
|
expect(results).to have_hits(1)
|
80
82
|
end
|
81
83
|
|
82
84
|
it "creates .keyword field from any string field which is not_analyzed" do
|
83
|
-
results = @es.search(:q
|
85
|
+
results = @es.search(index: 'logstash*', q: "country.keyword:\"us\"")
|
84
86
|
expect(results).to have_hits(1)
|
85
87
|
expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
|
86
88
|
|
87
89
|
# partial or terms should not work.
|
88
|
-
results = @es.search(:q
|
90
|
+
results = @es.search(index: 'logstash*', q: "country.keyword:\"u\"")
|
89
91
|
expect(results).to have_hits(0)
|
90
92
|
end
|
91
93
|
|
@@ -94,7 +96,7 @@ describe "index template expected behavior", :integration => true do
|
|
94
96
|
end
|
95
97
|
|
96
98
|
it "aggregate .keyword results correctly " do
|
97
|
-
results = @es.search(:body
|
99
|
+
results = @es.search(index: 'logstash*', body: { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
|
98
100
|
terms = results["buckets"].collect { |b| b["key"] }
|
99
101
|
|
100
102
|
expect(terms).to include("us")
|
@@ -114,7 +114,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
114
114
|
{
|
115
115
|
'hosts' => [ 'http://127.0.0.1:12345' ],
|
116
116
|
'http_compression' => 'true', 'bulk_path' => '_bulk', 'timeout' => '30',
|
117
|
-
'user' => 'elastic', 'password' => 'ForSearch!', '
|
117
|
+
'user' => 'elastic', 'password' => 'ForSearch!', 'ssl_enabled' => 'false'
|
118
118
|
}
|
119
119
|
end
|
120
120
|
|
@@ -164,7 +164,9 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
164
164
|
'data_stream_dataset' => 'any',
|
165
165
|
'data_stream_namespace' => 'any',
|
166
166
|
'data_stream_sync_fields' => true,
|
167
|
-
'data_stream_auto_routing' => true
|
167
|
+
'data_stream_auto_routing' => true,
|
168
|
+
'dlq_custom_codes' => [ 404 ],
|
169
|
+
'dlq_on_failed_indexname_interpolation' => false)
|
168
170
|
}
|
169
171
|
|
170
172
|
it 'should enable data-streams by default' do
|
@@ -50,6 +50,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
50
50
|
|
51
51
|
describe "healthcheck url handling" do
|
52
52
|
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
53
|
+
let(:success_response) { double("Response", :code => 200) }
|
54
|
+
|
53
55
|
before(:example) do
|
54
56
|
expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
|
55
57
|
expect(url.path).to be_empty
|
@@ -60,6 +62,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
60
62
|
it "performs the healthcheck to the root" do
|
61
63
|
expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
|
62
64
|
expect(url.path).to be_empty
|
65
|
+
|
66
|
+
success_response
|
63
67
|
end
|
64
68
|
expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
|
65
69
|
end
|
@@ -71,6 +75,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
71
75
|
it "performs the healthcheck to the healthcheck_path" do
|
72
76
|
expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
|
73
77
|
expect(url.path).to be_empty
|
78
|
+
|
79
|
+
success_response
|
74
80
|
end
|
75
81
|
expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
|
76
82
|
end
|
@@ -197,9 +203,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
197
203
|
"build_flavor" => 'default'}
|
198
204
|
})
|
199
205
|
end
|
206
|
+
let(:success_response) { double("head_req", :code => 200)}
|
200
207
|
|
201
208
|
before :each do
|
202
|
-
allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
|
209
|
+
allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil).and_return(success_response)
|
203
210
|
allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
|
204
211
|
end
|
205
212
|
let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
|
@@ -225,7 +232,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
225
232
|
u,m = subject.get_connection
|
226
233
|
|
227
234
|
# The resurrectionist will call this to check on the backend
|
228
|
-
response = double("response")
|
235
|
+
response = double("response", :code => 200)
|
229
236
|
expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
|
230
237
|
|
231
238
|
subject.return_connection(u)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
require_relative "../../../../spec/spec_helper"
|
2
2
|
require "logstash/outputs/elasticsearch/template_manager"
|
3
3
|
|
4
4
|
describe LogStash::Outputs::ElasticSearch::TemplateManager do
|
@@ -33,33 +33,85 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
-
|
36
|
+
context "index template with ilm settings" do
|
37
37
|
let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
|
38
38
|
let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
|
39
39
|
|
40
|
-
describe "
|
41
|
-
let(:file_path) { described_class.default_template_path(8) }
|
42
|
-
let(:template) { described_class.read_template_file(file_path)}
|
40
|
+
describe "with custom template" do
|
43
41
|
|
44
|
-
|
45
|
-
|
46
|
-
described_class.
|
47
|
-
|
48
|
-
|
49
|
-
|
42
|
+
describe "in version 8+" do
|
43
|
+
let(:file_path) { described_class.default_template_path(8) }
|
44
|
+
let(:template) { described_class.read_template_file(file_path)}
|
45
|
+
|
46
|
+
it "should update settings" do
|
47
|
+
expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
|
48
|
+
described_class.add_ilm_settings_to_template(plugin, template)
|
49
|
+
expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
|
50
|
+
expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
|
51
|
+
expect(template.include?('settings')).to be_falsey
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe "in version < 8" do
|
56
|
+
let(:file_path) { described_class.default_template_path(7) }
|
57
|
+
let(:template) { described_class.read_template_file(file_path)}
|
58
|
+
|
59
|
+
it "should update settings" do
|
60
|
+
expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
|
61
|
+
described_class.add_ilm_settings_to_template(plugin, template)
|
62
|
+
expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
|
63
|
+
expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
|
64
|
+
expect(template.include?('template')).to be_falsey
|
65
|
+
end
|
50
66
|
end
|
51
67
|
end
|
52
68
|
|
53
|
-
|
54
|
-
let(:
|
55
|
-
|
69
|
+
context "resolve template setting" do
|
70
|
+
let(:plugin_settings) { super().merge({"template_api" => template_api}) }
|
71
|
+
|
72
|
+
describe "with composable template API" do
|
73
|
+
let(:template_api) { "composable" }
|
56
74
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
75
|
+
it 'resolves composable index template API compatible setting' do
|
76
|
+
expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log
|
77
|
+
template = {}
|
78
|
+
described_class.resolve_template_settings(plugin, template)
|
79
|
+
expect(template["template"]["settings"]).not_to eq(nil)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
describe "with legacy template API" do
|
84
|
+
let(:template_api) { "legacy" }
|
85
|
+
|
86
|
+
it 'resolves legacy index template API compatible setting' do
|
87
|
+
expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log
|
88
|
+
template = {}
|
89
|
+
described_class.resolve_template_settings(plugin, template)
|
90
|
+
expect(template["settings"]).not_to eq(nil)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
describe "with `template_api => 'auto'`" do
|
95
|
+
let(:template_api) { "auto" }
|
96
|
+
|
97
|
+
describe "with ES < 8 versions" do
|
98
|
+
|
99
|
+
it 'resolves legacy index template API compatible setting' do
|
100
|
+
expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
|
101
|
+
template = {}
|
102
|
+
described_class.resolve_template_settings(plugin, template)
|
103
|
+
expect(template["settings"]).not_to eq(nil)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
describe "with ES >= 8 versions" do
|
108
|
+
it 'resolves composable index template API compatible setting' do
|
109
|
+
expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
|
110
|
+
template = {}
|
111
|
+
described_class.resolve_template_settings(plugin, template)
|
112
|
+
expect(template["template"]["settings"]).not_to eq(nil)
|
113
|
+
end
|
114
|
+
end
|
63
115
|
end
|
64
116
|
end
|
65
117
|
end
|