logstash-output-elasticsearch 10.4.0-java → 10.6.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (22) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +12 -0
  3. data/docs/index.asciidoc +90 -35
  4. data/lib/logstash/outputs/elasticsearch.rb +45 -0
  5. data/lib/logstash/outputs/elasticsearch/common.rb +29 -21
  6. data/lib/logstash/outputs/elasticsearch/common_configs.rb +3 -3
  7. data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +11 -1
  8. data/lib/logstash/outputs/elasticsearch/ilm.rb +1 -1
  9. data/lib/logstash/outputs/elasticsearch/template_manager.rb +12 -9
  10. data/lib/logstash/outputs/elasticsearch/{elasticsearch-template-es2x.json → templates/ecs-disabled/elasticsearch-2x.json} +0 -0
  11. data/lib/logstash/outputs/elasticsearch/{elasticsearch-template-es5x.json → templates/ecs-disabled/elasticsearch-5x.json} +0 -0
  12. data/lib/logstash/outputs/elasticsearch/{elasticsearch-template-es6x.json → templates/ecs-disabled/elasticsearch-6x.json} +0 -0
  13. data/lib/logstash/outputs/elasticsearch/{elasticsearch-template-es7x.json → templates/ecs-disabled/elasticsearch-7x.json} +0 -0
  14. data/lib/logstash/outputs/elasticsearch/{elasticsearch-template-es8x.json → templates/ecs-disabled/elasticsearch-8x.json} +0 -0
  15. data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json +2950 -0
  16. data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json +2948 -0
  17. data/logstash-output-elasticsearch.gemspec +3 -1
  18. data/spec/integration/outputs/ilm_spec.rb +2 -2
  19. data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +9 -3
  20. data/spec/unit/outputs/elasticsearch_spec.rb +86 -2
  21. data/spec/unit/outputs/error_whitelist_spec.rb +1 -1
  22. metadata +23 -7
@@ -1,6 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '10.4.0'
3
+ s.version = '10.6.0'
4
+
4
5
  s.licenses = ['apache-2.0']
5
6
  s.summary = "Stores logs in Elasticsearch"
6
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,6 +25,7 @@ Gem::Specification.new do |s|
24
25
  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
25
26
  s.add_runtime_dependency 'cabin', ['~> 0.6']
26
27
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
28
+ s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
27
29
 
28
30
  s.add_development_dependency 'logstash-codec-plain'
29
31
  s.add_development_dependency 'logstash-devutils'
@@ -249,9 +249,9 @@ if ESHelper.es_version_satisfies?(">= 6.6")
249
249
  }
250
250
  let (:small_max_doc_policy) { max_docs_policy(3) }
251
251
  let (:large_max_doc_policy) { max_docs_policy(1000000) }
252
- let (:expected_index) { LogStash::Outputs::ElasticSearch::DEFAULT_ROLLOVER_ALIAS }
252
+ let (:expected_index) { elasticsearch_output_plugin.default_ilm_rollover_alias }
253
253
 
254
- subject { LogStash::Outputs::ElasticSearch.new(settings) }
254
+ subject(:elasticsearch_output_plugin) { LogStash::Outputs::ElasticSearch.new(settings) }
255
255
 
256
256
  before :each do
257
257
  # Delete all templates first.
@@ -8,18 +8,24 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
8
8
  describe ".default_template_path" do
9
9
  context "elasticsearch 1.x" do
10
10
  it "chooses the 2x template" do
11
- expect(described_class.default_template_path(1)).to match(/elasticsearch-template-es2x.json/)
11
+ expect(described_class.default_template_path(1)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
12
12
  end
13
13
  end
14
14
  context "elasticsearch 2.x" do
15
15
  it "chooses the 2x template" do
16
- expect(described_class.default_template_path(2)).to match(/elasticsearch-template-es2x.json/)
16
+ expect(described_class.default_template_path(2)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
17
17
  end
18
18
  end
19
19
  context "elasticsearch 5.x" do
20
20
  it "chooses the 5x template" do
21
- expect(described_class.default_template_path(5)).to match(/elasticsearch-template-es5x.json/)
21
+ expect(described_class.default_template_path(5)).to end_with("/templates/ecs-disabled/elasticsearch-5x.json")
22
22
  end
23
23
  end
24
24
  end
25
+
26
+ context 'when ECS v1 is requested' do
27
+ it 'resolves' do
28
+ expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/elasticsearch-7x.json")
29
+ end
30
+ end
25
31
  end
@@ -1,11 +1,12 @@
1
1
  require_relative "../../../spec/es_spec_helper"
2
+ require "base64"
2
3
  require "flores/random"
3
4
  require "logstash/outputs/elasticsearch"
4
5
 
5
6
  describe LogStash::Outputs::ElasticSearch do
6
7
  subject { described_class.new(options) }
7
8
  let(:options) { {} }
8
- let(:maximum_seen_major_version) { rand(100) }
9
+ let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample }
9
10
 
10
11
  let(:do_register) { true }
11
12
 
@@ -142,6 +143,25 @@ describe LogStash::Outputs::ElasticSearch do
142
143
 
143
144
  include_examples("an authenticated config")
144
145
  end
146
+
147
+ context 'claud_auth also set' do
148
+ let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
149
+ let(:options) { { "user" => user, "password" => password, "cloud_auth" => "elastic:my-passwd-00" } }
150
+
151
+ it "should fail" do
152
+ expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
153
+ end
154
+ end
155
+
156
+ context 'api_key also set' do
157
+ let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
158
+ let(:options) { { "user" => user, "password" => password, "api_key" => "some_key" } }
159
+
160
+ it "should fail" do
161
+ expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
162
+ end
163
+ end
164
+
145
165
  end
146
166
 
147
167
  describe "with path" do
@@ -577,7 +597,15 @@ describe LogStash::Outputs::ElasticSearch do
577
597
  let(:options) { { 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' } }
578
598
 
579
599
  it "should fail" do
580
- expect { subject.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
600
+ expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
601
+ end
602
+ end
603
+
604
+ context 'api_key also set' do
605
+ let(:options) { { 'cloud_auth' => 'elastic:my-passwd-00', 'api_key' => 'some_key' } }
606
+
607
+ it "should fail" do
608
+ expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
581
609
  end
582
610
  end
583
611
  end if LOGSTASH_VERSION > '6.0'
@@ -659,6 +687,62 @@ describe LogStash::Outputs::ElasticSearch do
659
687
  end
660
688
  end
661
689
 
690
+ describe "API key" do
691
+ let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) }
692
+ let(:api_key) { "some_id:some_api_key" }
693
+ let(:base64_api_key) { "ApiKey c29tZV9pZDpzb21lX2FwaV9rZXk=" }
694
+
695
+ context "when set without ssl" do
696
+ let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
697
+ let(:options) { { "api_key" => api_key } }
698
+
699
+ it "should raise a configuration error" do
700
+ expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
701
+ end
702
+ end
703
+
704
+ context "when set without ssl but with a https host" do
705
+ let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
706
+ let(:options) { { "hosts" => ["https://some.host.com"], "api_key" => api_key } }
707
+
708
+ it "should raise a configuration error" do
709
+ expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
710
+ end
711
+ end
712
+
713
+ context "when set" do
714
+ let(:options) { { "ssl" => true, "api_key" => ::LogStash::Util::Password.new(api_key) } }
715
+
716
+ it "should use the custom headers in the adapter options" do
717
+ expect(manticore_options[:headers]).to eq({ "Authorization" => base64_api_key })
718
+ end
719
+ end
720
+
721
+ context "when not set" do
722
+ it "should have no headers" do
723
+ expect(manticore_options[:headers]).to be_empty
724
+ end
725
+ end
726
+
727
+ context 'user also set' do
728
+ let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
729
+ let(:options) { { "ssl" => true, "api_key" => api_key, 'user' => 'another' } }
730
+
731
+ it "should fail" do
732
+ expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
733
+ end
734
+ end
735
+
736
+ context 'cloud_auth also set' do
737
+ let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
738
+ let(:options) { { "ssl" => true, "api_key" => api_key, 'cloud_auth' => 'foobar' } }
739
+
740
+ it "should fail" do
741
+ expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
742
+ end
743
+ end
744
+ end
745
+
662
746
  @private
663
747
 
664
748
  def stub_manticore_client!(manticore_double = nil)
@@ -11,10 +11,10 @@ describe "whitelisting error types in expected behavior" do
11
11
 
12
12
  before :each do
13
13
  allow(subject.logger).to receive(:warn)
14
+ allow(subject).to receive(:maximum_seen_major_version).and_return(0)
14
15
 
15
16
  subject.register
16
17
 
17
- allow(subject.client).to receive(:maximum_seen_major_version).and_return(0)
18
18
  allow(subject.client).to receive(:get_xpack_info)
19
19
  allow(subject.client).to receive(:bulk).and_return(
20
20
  {
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 10.4.0
4
+ version: 10.6.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-26 00:00:00.000000000 Z
11
+ date: 2020-07-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,20 @@ dependencies:
84
84
  - - "<="
85
85
  - !ruby/object:Gem::Version
86
86
  version: '2.99'
87
+ - !ruby/object:Gem::Dependency
88
+ requirement: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - "~>"
91
+ - !ruby/object:Gem::Version
92
+ version: '1.0'
93
+ name: logstash-mixin-ecs_compatibility_support
94
+ prerelease: false
95
+ type: :runtime
96
+ version_requirements: !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - "~>"
99
+ - !ruby/object:Gem::Version
100
+ version: '1.0'
87
101
  - !ruby/object:Gem::Dependency
88
102
  requirement: !ruby/object:Gem::Requirement
89
103
  requirements:
@@ -159,17 +173,19 @@ files:
159
173
  - lib/logstash/outputs/elasticsearch/common.rb
160
174
  - lib/logstash/outputs/elasticsearch/common_configs.rb
161
175
  - lib/logstash/outputs/elasticsearch/default-ilm-policy.json
162
- - lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json
163
- - lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json
164
- - lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json
165
- - lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json
166
- - lib/logstash/outputs/elasticsearch/elasticsearch-template-es8x.json
167
176
  - lib/logstash/outputs/elasticsearch/http_client.rb
168
177
  - lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb
169
178
  - lib/logstash/outputs/elasticsearch/http_client/pool.rb
170
179
  - lib/logstash/outputs/elasticsearch/http_client_builder.rb
171
180
  - lib/logstash/outputs/elasticsearch/ilm.rb
172
181
  - lib/logstash/outputs/elasticsearch/template_manager.rb
182
+ - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json
183
+ - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json
184
+ - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json
185
+ - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json
186
+ - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
187
+ - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
188
+ - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
173
189
  - logstash-output-elasticsearch.gemspec
174
190
  - spec/es_spec_helper.rb
175
191
  - spec/fixtures/_nodes/2x_1x.json