logstash-output-elasticsearch 11.10.0-java → 11.12.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88cb9349aee0722c0f5c9b936257fa46a5954a9fc603d783e275206217a4f0af
-  data.tar.gz: f99d4aeed3b63a3c320ae058c1a3842dcedfee54f4064387cc61836396040220
+  metadata.gz: 497559fc1591316e5d904dfdfa907c63397a074a32cb54e4d249bb1e678a4d94
+  data.tar.gz: 0cab607b8ed00108dc733658a27e7e7a3e7a384d70ac9a69ebff7069a6d8a629
 SHA512:
-  metadata.gz: 98bcaa84acaa9ebcf13f8a6c64c9d40e220838adf5e321fcf198ff1a51ad016855172b119e4003c7fb13c204c156c77cf10d6a01f7fc574e8c4b20a0284f4391
-  data.tar.gz: 75827edd0dca20dd1e534e2abd3f4b1bf4e2174865d1fcdc677b24131c28a490267989f48ded95587af103a38486b6d524da1c81ce41f44ad347dcd6b40cedb0
+  metadata.gz: 3ce4e493f7b1951fc69ceb16e8f3370423e57ebfb19f069c14b41172a2a6597940eafd721878754649779e5c93f906dd21cb58c0c26689ff393773b7e866e9d5
+  data.tar.gz: d7299fc74267f8ba104842bd47493a744f0e683ab7ec51fe241cd4f53c6e5c63a6e597acbadc0174d661e630370d342b4c84581774ece082b2cc3b426aab93f0

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.12.0
+ - Add legacy template API support for Elasticsearch 8 [#1092](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1092)
+
+## 11.11.0
+ - When using an `api_key` along with either `cloud_id` or https `hosts`, you no longer need to also specify `ssl => true` [#1065](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1065)
+
 ## 11.10.0
  - Feature: expose `dlq_routed` document metric to track the documents routed into DLQ [#1090](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1090)
 

data/docs/index.asciidoc

@@ -362,6 +362,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-template_api>> |<<string,string>>, one of `["auto", "legacy", "composable"]`|No
 | <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
@@ -406,8 +407,8 @@ For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bu
   * Value type is <<password,password>>
   * There is no default value for this setting.
 
-Authenticate using Elasticsearch API key. Note that this option also requires
-enabling the `ssl` option.
+Authenticate using Elasticsearch API key.
+Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl,`ssl => true`>>.
 
 Format is `id:api_key` where `id` and `api_key` are as returned by the
 Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
@@ -1040,11 +1041,9 @@ do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
   * Value type is <<boolean,boolean>>
   * There is no default value for this setting.
 
-Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this
-unspecified will use whatever scheme is specified in the URLs listed in 'hosts'.
-If no explicit protocol is specified plain HTTP will be used. If SSL is
-explicitly disabled here the plugin will refuse to start if an HTTPS URL is
-given in 'hosts'
+Enable SSL/TLS secured communication to Elasticsearch cluster.
+Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
+If no explicit protocol is specified plain HTTP will be used.
 
 [id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
 ===== `ssl_certificate_verification`
@@ -1082,6 +1081,22 @@ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.
 You can set the path to your own template here, if you so desire.
 If not set, the included template will be used.
 
+[id="plugins-{type}s-{plugin}-template_api"]
+===== `template_api`
+
+  * Value can be any of: `auto`, `legacy`, `composable`
+  * Default value is `auto`
+
+The default setting of `auto` will use
+{ref}/index-templates.html[index template API] to create index template, if the
+Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher,
+and use {ref}/indices-templates-v1.html[legacy template API] otherwise.
+
+Setting this flag to `legacy` will use legacy template API to create index template.
+Setting this flag to `composable` will use index template API to create index template.
+
+NOTE: The format of template provided to <<plugins-{type}s-{plugin}-template>> needs to match the template API being used.
+
 [id="plugins-{type}s-{plugin}-template_name"]
 ===== `template_name`
 

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -77,12 +77,12 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def template_install(name, template, force=false)
-      if template_exists?(name) && !force
+    def template_install(template_endpoint, name, template, force=false)
+      if template_exists?(template_endpoint, name) && !force
         @logger.debug("Found existing Elasticsearch template, skipping template management", name: name)
         return
       end
-      template_put(name, template)
+      template_put(template_endpoint, name, template)
     end
 
     def last_es_version
@@ -402,20 +402,16 @@ module LogStash; module Outputs; class ElasticSearch;
       response.code >= 200 && response.code <= 299
     end
 
-    def template_exists?(name)
+    def template_exists?(template_endpoint, name)
       exists?("/#{template_endpoint}/#{name}")
     end
 
-    def template_put(name, template)
+    def template_put(template_endpoint, name, template)
       path = "#{template_endpoint}/#{name}"
       logger.info("Installing Elasticsearch template", name: name)
       @pool.put(path, nil, LogStash::Json.dump(template))
     end
 
-    def template_endpoint
-      maximum_seen_major_version < 8 ? '_template' : '_index_template'
-    end
-
     # ILM methods
 
     # check whether rollover alias already exists

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -1,8 +1,20 @@
 module LogStash; module Outputs; class ElasticSearch
   class TemplateManager
+    LEGACY_TEMPLATE_ENDPOINT = '_template'.freeze
+    INDEX_TEMPLATE_ENDPOINT = '_index_template'.freeze
+
     # To be mixed into the elasticsearch plugin base
     def self.install_template(plugin)
       return unless plugin.manage_template
+
+      if plugin.maximum_seen_major_version < 8 && plugin.template_api == 'auto'
+        plugin.logger.warn("`template_api => auto` resolved to `legacy` since we are connected to " + "Elasticsearch #{plugin.maximum_seen_major_version}, " +
+                           "but will resolve to `composable` the first time it connects to Elasticsearch 8+. " +
+                           "We recommend either setting `template_api => legacy` to continue providing legacy-style templates, " +
+                           "or migrating your template to the composable style and setting `template_api => composable`. " +
+                           "The legacy template API is slated for removal in Elasticsearch 9.")
+      end
+
       if plugin.template
         plugin.logger.info("Using mapping template from", :path => plugin.template)
         template = read_template_file(plugin.template)
@@ -14,7 +26,7 @@ module LogStash; module Outputs; class ElasticSearch
 
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
       plugin.logger.debug("Attempting to install template", template: template)
-      install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
+      install(plugin.client, template_endpoint(plugin), template_name(plugin), template, plugin.template_overwrite)
     end
 
     private
@@ -25,8 +37,8 @@ module LogStash; module Outputs; class ElasticSearch
       fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
     end
 
-    def self.install(client, template_name, template, template_overwrite)
-      client.template_install(template_name, template, template_overwrite)
+    def self.install(client, template_endpoint, template_name, template, template_overwrite)
+      client.template_install(template_endpoint, template_name, template, template_overwrite)
     end
 
     def self.add_ilm_settings_to_template(plugin, template)
@@ -63,5 +75,16 @@ module LogStash; module Outputs; class ElasticSearch
       template_data = ::IO.read(template_path)
       LogStash::Json.load(template_data)
     end
+
+    def self.template_endpoint(plugin)
+      if plugin.template_api == 'auto'
+        plugin.maximum_seen_major_version < 8 ? LEGACY_TEMPLATE_ENDPOINT : INDEX_TEMPLATE_ENDPOINT
+      elsif plugin.template_api.to_s == 'legacy'
+        LEGACY_TEMPLATE_ENDPOINT
+      else
+        INDEX_TEMPLATE_ENDPOINT
+      end
+    end
+
   end
 end end end

data/lib/logstash/outputs/elasticsearch.rb

@@ -185,6 +185,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # the "logstash" template (i.e. removing all customized settings)
   config :template_overwrite, :validate => :boolean, :default => false
 
+  # Flag for enabling legacy template api for Elasticsearch 8
+  # Default auto will use index template api for Elasticsearch 8 and use legacy api for 7
+  # Set to legacy to use legacy template api
+  config :template_api, :validate => ['auto', 'legacy', 'composable'], :default => 'auto'
+
   # The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
   # See https://www.elastic.co/blog/elasticsearch-versioning-support.
   config :version, :validate => :string

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -23,10 +23,14 @@ module LogStash; module PluginMixins; module ElasticSearch
       # because they must be executed prior to building the client and logstash
       # monitoring and management rely on directly calling build_client
       # see https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/934#pullrequestreview-396203307
-      validate_authentication
       fill_hosts_from_cloud_id
+      validate_authentication
+
       setup_hosts
 
+
+      params['ssl'] = effectively_ssl? unless params.include?('ssl')
+
       # inject the TrustStrategy from CATrustedFingerprintSupport
       if trust_strategy_for_ca_trusted_fingerprint
         params["ssl_trust_strategy"] = trust_strategy_for_ca_trusted_fingerprint
@@ -49,7 +53,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
       end
 
-      if @api_key && @api_key.value && @ssl   != true
+      if @api_key && @api_key.value && !effectively_ssl?
         raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
       end
 
@@ -69,6 +73,15 @@ module LogStash; module PluginMixins; module ElasticSearch
       end
     end
 
+    def effectively_ssl?
+      return @ssl unless @ssl.nil?
+
+      hosts = Array(@hosts)
+      return false if hosts.nil? || hosts.empty?
+
+      hosts.all? { |host| host && host.scheme == "https" }
+    end
+
     def hosts_default?(hosts)
       # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
       hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(LogStash::PluginMixins::ElasticSearch::APIConfigs::DEFAULT_HOST)
@@ -208,12 +221,12 @@ module LogStash; module PluginMixins; module ElasticSearch
 
     def handle_dlq_response(message, action, status, response)
       _, action_params = action.event, [action[0], action[1], action[2]]
-      
+
       # TODO: Change this to send a map with { :status => status, :action => action } in the future
       detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
-      
+
       log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
-      
+
       handle_dlq_status(action.event, log_level, detailed_message)
     end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.10.0'
+  s.version         = '11.12.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -140,27 +140,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     end
   end
 
-  describe "index template" do
-    subject { described_class.new(base_options) }
-    let(:template_name) { "logstash" }
-    let(:template) { {} }
-    let(:get_response) {
-      double("response", :body => {})
-    }
-
-    it "should call composable index template in version 8+" do
-      expect(subject).to receive(:maximum_seen_major_version).and_return(8)
-      expect(subject.pool).to receive(:put).with("_index_template/#{template_name}", nil, anything).and_return(get_response)
-      subject.template_put(template_name, template)
-    end
-
-    it "should call index template in version < 8" do
-      expect(subject).to receive(:maximum_seen_major_version).and_return(7)
-      expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
-      subject.template_put(template_name, template)
-    end
-  end
-
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -63,4 +63,53 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
       end
     end
   end
+
+  describe "template endpoint" do
+    describe "template_api => 'auto'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'auto'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use index template API" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
+        end
+      end
+
+      describe "in version < 8" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+
+    describe "template_api => 'legacy'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'legacy'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:maximum_seen_major_version).never
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+
+    describe "template_api => 'composable'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'composable'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:maximum_seen_major_version).never
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager:: INDEX_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+  end
 end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -17,12 +17,17 @@ describe LogStash::Outputs::ElasticSearch do
     allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:start)
   end
 
+  let(:spy_http_client_builder!) do
+    allow(described_class::HttpClientBuilder).to receive(:build).with(any_args).and_call_original
+  end
+
   let(:after_successful_connection_thread_mock) do
     double('after_successful_connection_thread', value: true)
   end
 
   before(:each) do
     if do_register
+      spy_http_client_builder!
       stub_http_client_pool!
 
       allow(subject).to receive(:finish_register) # stub-out thread completion (to avoid error log entries)
@@ -1003,29 +1008,88 @@ describe LogStash::Outputs::ElasticSearch do
     let(:api_key) { "some_id:some_api_key" }
     let(:base64_api_key) { "ApiKey c29tZV9pZDpzb21lX2FwaV9rZXk=" }
 
-    context "when set without ssl" do
+    shared_examples 'secure api-key authenticated client' do
+      let(:do_register) { true }
+
+      it 'adds the appropriate Authorization header to the manticore client' do
+        expect(manticore_options[:headers]).to eq({ "Authorization" => base64_api_key })
+      end
+      it 'is provides ssl=>true to the http client builder' do; aggregate_failures do
+        expect(described_class::HttpClientBuilder).to have_received(:build).with(anything, anything, hash_including('ssl'=>true))
+      end; end
+    end
+
+    context "when set without ssl => true" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "api_key" => api_key } }
 
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
       end
+
+      context 'with cloud_id' do
+        let(:sample_cloud_id) { 'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==' }
+        let(:options) { super().merge('cloud_id' => sample_cloud_id) }
+
+        it_behaves_like 'secure api-key authenticated client'
+      end
     end
 
-    context "when set without ssl but with a https host" do
+    context "when set without ssl specified but with an https host" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "hosts" => ["https://some.host.com"], "api_key" => api_key } }
 
+      it_behaves_like 'secure api-key authenticated client'
+    end
+
+    context "when set without ssl specified but with an http host`" do
+      let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
+      let(:options) { { "hosts" => ["http://some.host.com"], "api_key" => api_key } }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
+      end
+    end
+
+    context "when set with `ssl => false`" do
+      let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
+      let(:options) { { "ssl" => "false", "api_key" => api_key } }
+
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
       end
     end
 
     context "when set" do
-      let(:options) { { "ssl" => true, "api_key" =>  ::LogStash::Util::Password.new(api_key) } }
+      let(:options) { { "api_key" => ::LogStash::Util::Password.new(api_key) } }
 
-      it "should use the custom headers in the adapter options" do
-        expect(manticore_options[:headers]).to eq({ "Authorization" => base64_api_key })
+      context "with ssl => true" do
+        let(:options) { super().merge("ssl" => true) }
+        it_behaves_like 'secure api-key authenticated client'
+      end
+
+      context "with ssl => false" do
+        let(:options) { super().merge("ssl" => "false") }
+
+        let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
+        it "should raise a configuration error" do
+          expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
+        end
+      end
+
+      context "without ssl specified" do
+        context "with an https host" do
+          let(:options) { super().merge("hosts" => ["https://some.host.com"]) }
+          it_behaves_like 'secure api-key authenticated client'
+        end
+        context "with an http host`" do
+          let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
+          let(:options) { { "hosts" => ["http://some.host.com"], "api_key" => api_key } }
+
+          it "should raise a configuration error" do
+            expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
+          end
+        end
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.10.0
+  version: 11.12.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-10 00:00:00.000000000 Z
+date: 2022-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement