logstash-output-elasticsearch 11.3.0-java → 11.4.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -2
- data/docs/index.asciidoc +3 -7
- data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +43 -9
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +40 -16
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +5 -9
- data/lib/logstash/outputs/elasticsearch/http_client.rb +1 -2
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json +2196 -288
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json +2196 -288
- data/lib/logstash/outputs/elasticsearch.rb +5 -1
- data/logstash-output-elasticsearch.gemspec +2 -1
- data/spec/es_spec_helper.rb +5 -1
- data/spec/integration/outputs/data_stream_spec.rb +7 -0
- data/spec/integration/outputs/ilm_spec.rb +2 -1
- data/spec/integration/outputs/ingest_pipeline_spec.rb +2 -1
- data/spec/integration/outputs/retry_spec.rb +2 -1
- data/spec/integration/outputs/sniffer_spec.rb +1 -1
- data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +52 -4
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +6 -5
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +1 -1
- data/spec/unit/outputs/elasticsearch_spec.rb +22 -1
- metadata +16 -2
@@ -94,6 +94,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
94
94
|
require "logstash/outputs/elasticsearch/ilm"
|
95
95
|
require "logstash/outputs/elasticsearch/data_stream_support"
|
96
96
|
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
97
|
+
require 'logstash/plugin_mixins/deprecation_logger_support'
|
97
98
|
|
98
99
|
# Protocol agnostic methods
|
99
100
|
include(LogStash::PluginMixins::ElasticSearch::Common)
|
@@ -104,6 +105,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
104
105
|
# ecs_compatibility option, provided by Logstash core or the support adapter.
|
105
106
|
include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
|
106
107
|
|
108
|
+
# deprecation logger adapter for older Logstashes
|
109
|
+
include(LogStash::PluginMixins::DeprecationLoggerSupport)
|
110
|
+
|
107
111
|
# Generic/API config options that any document indexer output needs
|
108
112
|
include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
|
109
113
|
|
@@ -498,7 +502,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
498
502
|
@default_index = "logstash-%{+yyyy.MM.dd}"
|
499
503
|
@default_ilm_rollover_alias = "logstash"
|
500
504
|
@default_template_name = 'logstash'
|
501
|
-
when :v1
|
505
|
+
when :v1, :v8
|
502
506
|
@default_index = "ecs-logstash-%{+yyyy.MM.dd}"
|
503
507
|
@default_ilm_rollover_alias = "ecs-logstash"
|
504
508
|
@default_template_name = 'ecs-logstash'
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-elasticsearch'
|
3
|
-
s.version = '11.
|
3
|
+
s.version = '11.4.0'
|
4
4
|
|
5
5
|
s.licenses = ['apache-2.0']
|
6
6
|
s.summary = "Stores logs in Elasticsearch"
|
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
|
|
25
25
|
s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
|
26
26
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
27
27
|
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
|
28
|
+
s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
|
28
29
|
|
29
30
|
s.add_development_dependency 'logstash-codec-plain'
|
30
31
|
s.add_development_dependency 'logstash-devutils'
|
data/spec/es_spec_helper.rb
CHANGED
@@ -59,7 +59,11 @@ module ESHelper
|
|
59
59
|
end
|
60
60
|
|
61
61
|
def self.es_version
|
62
|
-
|
62
|
+
[
|
63
|
+
nilify(RSpec.configuration.filter[:es_version]),
|
64
|
+
nilify(ENV['ES_VERSION']),
|
65
|
+
nilify(ENV['ELASTIC_STACK_VERSION']),
|
66
|
+
].compact.first
|
63
67
|
end
|
64
68
|
|
65
69
|
RSpec::Matchers.define :have_hits do |expected|
|
@@ -12,6 +12,13 @@ describe "data streams", :integration => true do
|
|
12
12
|
|
13
13
|
subject { LogStash::Outputs::ElasticSearch.new(options) }
|
14
14
|
|
15
|
+
# All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
|
16
|
+
# We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
|
17
|
+
let(:ecs_compatibility) { :v1 }
|
18
|
+
before(:each) do
|
19
|
+
allow( subject ).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
20
|
+
end
|
21
|
+
|
15
22
|
before :each do
|
16
23
|
@es = get_client
|
17
24
|
@es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil
|
@@ -236,7 +236,8 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
|
|
236
236
|
let (:settings) {
|
237
237
|
{
|
238
238
|
"ilm_enabled" => ilm_enabled,
|
239
|
-
"hosts" => "#{get_host_port()}"
|
239
|
+
"hosts" => "#{get_host_port()}",
|
240
|
+
"ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
|
240
241
|
}
|
241
242
|
}
|
242
243
|
let (:small_max_doc_policy) { max_docs_policy(3) }
|
@@ -6,7 +6,8 @@ describe "Ingest pipeline execution behavior", :integration => true do
|
|
6
6
|
settings = {
|
7
7
|
"hosts" => "#{get_host_port()}",
|
8
8
|
"pipeline" => "apache-logs",
|
9
|
-
"data_stream" => 'false'
|
9
|
+
"data_stream" => 'false',
|
10
|
+
"ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
|
10
11
|
}
|
11
12
|
next LogStash::Outputs::ElasticSearch.new(settings)
|
12
13
|
end
|
@@ -45,7 +45,8 @@ describe "failures in bulk class expected behavior", :integration => true do
|
|
45
45
|
"template_overwrite" => true,
|
46
46
|
"hosts" => get_host_port(),
|
47
47
|
"retry_max_interval" => 64,
|
48
|
-
"retry_initial_interval" => 2
|
48
|
+
"retry_initial_interval" => 2,
|
49
|
+
"ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
|
49
50
|
}
|
50
51
|
next LogStash::Outputs::ElasticSearch.new(settings)
|
51
52
|
end
|
@@ -5,7 +5,7 @@ require "socket"
|
|
5
5
|
|
6
6
|
describe "pool sniffer", :integration => true do
|
7
7
|
let(:logger) { Cabin::Channel.get }
|
8
|
-
let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
|
8
|
+
let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
|
9
9
|
let(:es_host) { get_host_port.split(":").first }
|
10
10
|
let(:es_port) { get_host_port.split(":").last }
|
11
11
|
let(:es_ip) { IPSocket.getaddress(es_host) }
|
@@ -4,9 +4,17 @@ require "logstash/outputs/elasticsearch/data_stream_support"
|
|
4
4
|
describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
5
5
|
|
6
6
|
subject { LogStash::Outputs::ElasticSearch.new(options) }
|
7
|
+
|
7
8
|
let(:options) { { 'hosts' => [ 'localhost:12345' ] } }
|
8
9
|
let(:es_version) { '7.10.1' }
|
9
10
|
|
11
|
+
# All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
|
12
|
+
# We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
|
13
|
+
let(:ecs_compatibility) { :v1 }
|
14
|
+
before(:each) do
|
15
|
+
allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
16
|
+
end
|
17
|
+
|
10
18
|
let(:do_register) { false }
|
11
19
|
let(:stub_plugin_register!) do
|
12
20
|
allow(subject).to receive(:last_es_version).and_return(es_version)
|
@@ -52,9 +60,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
52
60
|
end
|
53
61
|
|
54
62
|
it "warns when configuration is data-stream compliant (LS 7.x)" do
|
55
|
-
expect( subject.logger ).to receive(:warn)
|
56
|
-
expect(msg).to include "Configuration is data stream compliant but due backwards compatibility Logstash 7.x"
|
57
|
-
end
|
63
|
+
expect( subject.logger ).to receive(:warn).with(a_string_including "Configuration is data stream compliant but due backwards compatibility Logstash 7.x")
|
58
64
|
change_constant :LOGSTASH_VERSION, '7.11.0' do
|
59
65
|
expect( subject.data_stream_config? ).to be false
|
60
66
|
end
|
@@ -66,6 +72,25 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
66
72
|
end
|
67
73
|
end
|
68
74
|
|
75
|
+
context 'ecs_compatibility disabled' do
|
76
|
+
let(:ecs_compatibility) { :disabled }
|
77
|
+
|
78
|
+
{
|
79
|
+
'7.x (pre-DS)' => '7.9.0',
|
80
|
+
'7.x (with DS)' => '7.11.0',
|
81
|
+
'8.0' => '8.0.0',
|
82
|
+
'8.x' => '8.1.2',
|
83
|
+
}.each do |ls_version_desc, ls_version|
|
84
|
+
context "on LS #{ls_version_desc}" do
|
85
|
+
around(:each) { |example| change_constant(:LOGSTASH_VERSION, ls_version, &example) }
|
86
|
+
it "does not use data-streams" do
|
87
|
+
expect( subject.logger ).to receive(:debug).with(a_string_including "ecs_compatibility is not enabled")
|
88
|
+
expect( subject.data_stream_config? ).to be false
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
69
94
|
context 'non-compatible ES' do
|
70
95
|
|
71
96
|
let(:es_version) { '7.8.0' }
|
@@ -95,6 +120,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
95
120
|
before { allow(subject).to receive(:last_es_version).and_return(es_version) }
|
96
121
|
|
97
122
|
it "does not use data-streams on LS 7.x" do
|
123
|
+
expect( subject.logger ).to receive(:warn).with(a_string_including "Logstash 7.x will not assume writing to a data-stream")
|
98
124
|
change_constant :LOGSTASH_VERSION, '7.10.0' do
|
99
125
|
expect( subject.data_stream_config? ).to be false
|
100
126
|
end
|
@@ -197,12 +223,34 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
197
223
|
end
|
198
224
|
end
|
199
225
|
|
200
|
-
it "does use data-streams on LS 8.
|
226
|
+
it "does use data-streams on LS 8.x" do
|
201
227
|
change_constant :LOGSTASH_VERSION, '8.1.0' do
|
202
228
|
expect( subject.data_stream_config? ).to be true
|
203
229
|
end
|
204
230
|
end
|
205
231
|
|
232
|
+
context 'with ecs_compatibility disabled' do
|
233
|
+
let(:ecs_compatibility) { :disabled }
|
234
|
+
|
235
|
+
context 'when running on LS 7.x' do
|
236
|
+
around(:each) { |example| change_constant(:LOGSTASH_VERSION, '7.15.1', &example) }
|
237
|
+
|
238
|
+
it "emits a deprecation warning and uses data streams anway" do
|
239
|
+
expect( subject.deprecation_logger ).to receive(:deprecated).with(a_string_including "`data_stream => true` will require the plugin to be run in ECS compatibility mode")
|
240
|
+
expect( subject.data_stream_config? ).to be true
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
context 'when running on LS 8.x' do
|
245
|
+
around(:each) { |example| change_constant(:LOGSTASH_VERSION, '8.0.0', &example) }
|
246
|
+
|
247
|
+
it "errors helpfully" do
|
248
|
+
expect{ subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, a_string_including("Invalid data stream configuration: `ecs_compatibility => disabled`"))
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
end
|
253
|
+
|
206
254
|
context 'non-compatible ES' do
|
207
255
|
|
208
256
|
let(:es_version) { '6.8.11' }
|
@@ -10,11 +10,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
|
|
10
10
|
|
11
11
|
it "should raise an exception if requests are issued after close" do
|
12
12
|
subject.close
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
13
|
+
begin
|
14
|
+
subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/')
|
15
|
+
fail 'expected to raise a HostUnreachableError'
|
16
|
+
rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
|
17
|
+
expect( e.original_error ).to be_a ::Manticore::ClientStoppedException
|
18
|
+
end
|
18
19
|
end
|
19
20
|
|
20
21
|
describe "auth" do
|
@@ -4,7 +4,7 @@ require 'cabin'
|
|
4
4
|
|
5
5
|
describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
6
6
|
let(:logger) { Cabin::Channel.get }
|
7
|
-
let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
|
7
|
+
let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
|
8
8
|
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
|
9
9
|
let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
|
10
10
|
let(:es_node_versions) { [ "0.0.0" ] }
|
@@ -4,6 +4,8 @@ require "flores/random"
|
|
4
4
|
require 'concurrent/atomic/count_down_latch'
|
5
5
|
require "logstash/outputs/elasticsearch"
|
6
6
|
|
7
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
|
8
|
+
|
7
9
|
describe LogStash::Outputs::ElasticSearch do
|
8
10
|
subject(:elasticsearch_output_instance) { described_class.new(options) }
|
9
11
|
let(:options) { {} }
|
@@ -922,6 +924,25 @@ describe LogStash::Outputs::ElasticSearch do
|
|
922
924
|
end
|
923
925
|
end
|
924
926
|
|
927
|
+
describe 'ECS Compatibility Support', :ecs_compatibility_support do
|
928
|
+
[
|
929
|
+
:disabled,
|
930
|
+
:v1,
|
931
|
+
:v8,
|
932
|
+
].each do |ecs_compatibility|
|
933
|
+
context "When initialized with `ecs_compatibility => #{ecs_compatibility}`" do
|
934
|
+
let(:options) { Hash.new }
|
935
|
+
subject(:output) { described_class.new(options.merge("ecs_compatibility" => "#{ecs_compatibility}")) }
|
936
|
+
context 'when registered' do
|
937
|
+
before(:each) { output.register }
|
938
|
+
it 'has the correct effective ECS compatibility setting' do
|
939
|
+
expect(output.ecs_compatibility).to eq(ecs_compatibility)
|
940
|
+
end
|
941
|
+
end
|
942
|
+
end
|
943
|
+
end
|
944
|
+
end
|
945
|
+
|
925
946
|
describe "post-register ES setup" do
|
926
947
|
let(:do_register) { false }
|
927
948
|
let(:es_version) { '7.10.0' } # DS default on LS 8.x
|
@@ -959,7 +980,7 @@ describe LogStash::Outputs::ElasticSearch do
|
|
959
980
|
context 'error raised' do
|
960
981
|
|
961
982
|
let(:es_version) { '7.8.0' }
|
962
|
-
let(:options) { super().merge('data_stream' => 'true') }
|
983
|
+
let(:options) { super().merge('data_stream' => 'true', 'ecs_compatibility' => 'v1') }
|
963
984
|
let(:latch) { Concurrent::CountDownLatch.new }
|
964
985
|
|
965
986
|
before do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 11.
|
4
|
+
version: 11.4.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,6 +84,20 @@ dependencies:
|
|
84
84
|
- - "~>"
|
85
85
|
- !ruby/object:Gem::Version
|
86
86
|
version: '1.0'
|
87
|
+
- !ruby/object:Gem::Dependency
|
88
|
+
requirement: !ruby/object:Gem::Requirement
|
89
|
+
requirements:
|
90
|
+
- - "~>"
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: '1.0'
|
93
|
+
name: logstash-mixin-deprecation_logger_support
|
94
|
+
prerelease: false
|
95
|
+
type: :runtime
|
96
|
+
version_requirements: !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - "~>"
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: '1.0'
|
87
101
|
- !ruby/object:Gem::Dependency
|
88
102
|
requirement: !ruby/object:Gem::Requirement
|
89
103
|
requirements:
|