logstash-output-elasticsearch 11.9.3-java → 11.11.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d02f6b7c9fca46bf86c64ad4dc6ec39f692899d30a069c9813a174be6b7436f
-  data.tar.gz: b68bdafacbbfa03c1eb0968ca111141353a1911a328144c62e9810de9902995c
+  metadata.gz: 6c35d86223e3353d75bc09c1e86ac45b164133ed877b64496e5734627f8bfe9b
+  data.tar.gz: 57bbcc13083b42c010c13fd51466ac9872ab09522ac4fb8e2833f429f1d18fbf
 SHA512:
-  metadata.gz: 2fef1eeb91cb331f340b36bbd76778f16dc1baedb3d550683b2ea2a8c454b1f87c65847b44ba9ae1323d9a413fe7610c510415d275a56a84c7ea743b9dbbfb23
-  data.tar.gz: 897c6b6d949c1fa5674faa745d4646c035058cc7ac84f58e69f0f1fe375f3374af8f07cce54ee534c6c2d2804ca6e87f8472d5b96cff0b8b1a38fc1009135fdb
+  metadata.gz: adfd2a4f7d288019fff8908bfd06a1e34ad5d07ca0bd1ca31d73d5394eddf2d8b0dfdbd0f8da983f5e66022d8d81c7852070be78914442cdc1b36fa7b67dc4ac
+  data.tar.gz: a99011dbe58769d6b9069c8ac5a4afc5b5a3682a6226d49b1d7b3d1ca687cbbabbfcd6816c68a27027b2b315a8501efdbba43013e980191b8cb5ac3ba78a56cb

data/CHANGELOG.md

@@ -1,8 +1,14 @@
+## 11.11.0
+ - When using an `api_key` along with either `cloud_id` or https `hosts`, you no longer need to also specify `ssl => true` [#1065](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1065)
+
+## 11.10.0
+ - Feature: expose `dlq_routed` document metric to track the documents routed into DLQ [#1090](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1090)
+
 ## 11.9.3
  - DOC: clarify that `http_compression` option only affects _requests_; compressed _responses_ have always been read independent of this setting [#1030 ](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1030)
 
 ## 11.9.2
-  - Fix broken link to Logstash Reference [#1085](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1085)
+ - Fix broken link to Logstash Reference [#1085](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1085)
 
 ## 11.9.1
  - Fixes a possible infinite-retry-loop that could occur when this plugin is configured with an `action` whose value contains a [sprintf-style placeholder][] that fails to be resolved for an individual event. Events in this state will be routed to the pipeline's [dead letter queue][DLQ] if it is available, or will be logged-and-dropped so that the remaining events in the batch can be processed [#1080](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1080)

data/docs/index.asciidoc

@@ -406,8 +406,8 @@ For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bu
   * Value type is <<password,password>>
   * There is no default value for this setting.
 
-Authenticate using Elasticsearch API key. Note that this option also requires
-enabling the `ssl` option.
+Authenticate using Elasticsearch API key.
+Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl,`ssl => true`>>.
 
 Format is `id:api_key` where `id` and `api_key` are as returned by the
 Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
@@ -1040,11 +1040,9 @@ do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
   * Value type is <<boolean,boolean>>
   * There is no default value for this setting.
 
-Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this
-unspecified will use whatever scheme is specified in the URLs listed in 'hosts'.
-If no explicit protocol is specified plain HTTP will be used. If SSL is
-explicitly disabled here the plugin will refuse to start if an HTTPS URL is
-given in 'hosts'
+Enable SSL/TLS secured communication to Elasticsearch cluster.
+Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
+If no explicit protocol is specified plain HTTP will be used.
 
 [id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
 ===== `ssl_certificate_verification`

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -23,10 +23,14 @@ module LogStash; module PluginMixins; module ElasticSearch
       # because they must be executed prior to building the client and logstash
       # monitoring and management rely on directly calling build_client
       # see https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/934#pullrequestreview-396203307
-      validate_authentication
       fill_hosts_from_cloud_id
+      validate_authentication
+
       setup_hosts
 
+
+      params['ssl'] = effectively_ssl? unless params.include?('ssl')
+
       # inject the TrustStrategy from CATrustedFingerprintSupport
       if trust_strategy_for_ca_trusted_fingerprint
         params["ssl_trust_strategy"] = trust_strategy_for_ca_trusted_fingerprint
@@ -49,7 +53,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
       end
 
-      if @api_key && @api_key.value && @ssl   != true
+      if @api_key && @api_key.value && !effectively_ssl?
         raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
       end
 
@@ -69,6 +73,15 @@ module LogStash; module PluginMixins; module ElasticSearch
       end
     end
 
+    def effectively_ssl?
+      return @ssl unless @ssl.nil?
+
+      hosts = Array(@hosts)
+      return false if hosts.nil? || hosts.empty?
+
+      hosts.all? { |host| host && host.scheme == "https" }
+    end
+
     def hosts_default?(hosts)
       # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
       hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(LogStash::PluginMixins::ElasticSearch::APIConfigs::DEFAULT_HOST)
@@ -208,12 +221,12 @@ module LogStash; module PluginMixins; module ElasticSearch
 
     def handle_dlq_response(message, action, status, response)
       _, action_params = action.event, [action[0], action[1], action[2]]
-      
+
       # TODO: Change this to send a map with { :status => status, :action => action } in the future
       detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
-      
+
       log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
-      
+
       handle_dlq_status(action.event, log_level, detailed_message)
     end
 
@@ -273,7 +286,7 @@ module LogStash; module PluginMixins; module ElasticSearch
           next
         elsif @dlq_codes.include?(status)
           handle_dlq_response("Could not index event to Elasticsearch.", action, status, response)
-          @document_level_metrics.increment(:non_retryable_failures)
+          @document_level_metrics.increment(:dlq_routed)
           next
         else
           # only log what the user whitelisted

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.9.3'
-
+  s.version         = '11.11.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/metrics_spec.rb

@@ -58,7 +58,7 @@ describe "metrics", :integration => true do
     end
 
     it "increases number of successful and non retryable documents" do
-      expect(document_level_metrics).to receive(:increment).with(:non_retryable_failures).once
+      expect(document_level_metrics).to receive(:increment).with(:dlq_routed).once
       expect(document_level_metrics).to receive(:increment).with(:successes).once
       subject.multi_receive(bulk)
     end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -17,12 +17,17 @@ describe LogStash::Outputs::ElasticSearch do
     allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:start)
   end
 
+  let(:spy_http_client_builder!) do
+    allow(described_class::HttpClientBuilder).to receive(:build).with(any_args).and_call_original
+  end
+
   let(:after_successful_connection_thread_mock) do
     double('after_successful_connection_thread', value: true)
   end
 
   before(:each) do
     if do_register
+      spy_http_client_builder!
       stub_http_client_pool!
 
       allow(subject).to receive(:finish_register) # stub-out thread completion (to avoid error log entries)
@@ -1003,29 +1008,88 @@ describe LogStash::Outputs::ElasticSearch do
     let(:api_key) { "some_id:some_api_key" }
     let(:base64_api_key) { "ApiKey c29tZV9pZDpzb21lX2FwaV9rZXk=" }
 
-    context "when set without ssl" do
+    shared_examples 'secure api-key authenticated client' do
+      let(:do_register) { true }
+
+      it 'adds the appropriate Authorization header to the manticore client' do
+        expect(manticore_options[:headers]).to eq({ "Authorization" => base64_api_key })
+      end
+      it 'is provides ssl=>true to the http client builder' do; aggregate_failures do
+        expect(described_class::HttpClientBuilder).to have_received(:build).with(anything, anything, hash_including('ssl'=>true))
+      end; end
+    end
+
+    context "when set without ssl => true" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "api_key" => api_key } }
 
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
       end
+
+      context 'with cloud_id' do
+        let(:sample_cloud_id) { 'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==' }
+        let(:options) { super().merge('cloud_id' => sample_cloud_id) }
+
+        it_behaves_like 'secure api-key authenticated client'
+      end
     end
 
-    context "when set without ssl but with a https host" do
+    context "when set without ssl specified but with an https host" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "hosts" => ["https://some.host.com"], "api_key" => api_key } }
 
+      it_behaves_like 'secure api-key authenticated client'
+    end
+
+    context "when set without ssl specified but with an http host`" do
+      let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
+      let(:options) { { "hosts" => ["http://some.host.com"], "api_key" => api_key } }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
+      end
+    end
+
+    context "when set with `ssl => false`" do
+      let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
+      let(:options) { { "ssl" => "false", "api_key" => api_key } }
+
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
       end
     end
 
     context "when set" do
-      let(:options) { { "ssl" => true, "api_key" =>  ::LogStash::Util::Password.new(api_key) } }
+      let(:options) { { "api_key" => ::LogStash::Util::Password.new(api_key) } }
 
-      it "should use the custom headers in the adapter options" do
-        expect(manticore_options[:headers]).to eq({ "Authorization" => base64_api_key })
+      context "with ssl => true" do
+        let(:options) { super().merge("ssl" => true) }
+        it_behaves_like 'secure api-key authenticated client'
+      end
+
+      context "with ssl => false" do
+        let(:options) { super().merge("ssl" => "false") }
+
+        let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
+        it "should raise a configuration error" do
+          expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
+        end
+      end
+
+      context "without ssl specified" do
+        context "with an https host" do
+          let(:options) { super().merge("hosts" => ["https://some.host.com"]) }
+          it_behaves_like 'secure api-key authenticated client'
+        end
+        context "with an http host`" do
+          let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
+          let(:options) { { "hosts" => ["http://some.host.com"], "api_key" => api_key } }
+
+          it "should raise a configuration error" do
+            expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
+          end
+        end
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.9.3
+  version: 11.11.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-04 00:00:00.000000000 Z
+date: 2022-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement