logstash-output-elasticsearch 11.11.0-java → 11.12.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c35d86223e3353d75bc09c1e86ac45b164133ed877b64496e5734627f8bfe9b
-  data.tar.gz: 57bbcc13083b42c010c13fd51466ac9872ab09522ac4fb8e2833f429f1d18fbf
+  metadata.gz: 484fe25cf95173ddd0bc6107393004e4b951b7302e01ef3941b8db7b2f948604
+  data.tar.gz: 0a63851b6e5623c2eaddbc4f3b225ec6f5a21f5c49cd3004adb753c77bb08099
 SHA512:
-  metadata.gz: adfd2a4f7d288019fff8908bfd06a1e34ad5d07ca0bd1ca31d73d5394eddf2d8b0dfdbd0f8da983f5e66022d8d81c7852070be78914442cdc1b36fa7b67dc4ac
-  data.tar.gz: a99011dbe58769d6b9069c8ac5a4afc5b5a3682a6226d49b1d7b3d1ca687cbbabbfcd6816c68a27027b2b315a8501efdbba43013e980191b8cb5ac3ba78a56cb
+  metadata.gz: db0556955eab387ce9f65784266dd11be8d1c8d2d23c10cb7fac51261068c20e12b335b4db92c8dd4741948fb34dd80be618a63245e3698357632e51e69ea3c5
+  data.tar.gz: 82dd5eac05b580e74e745c59204e100a463c03d16b40862e532acc0769e8f917cc576808dfbf85f2a2a9b9a892de0ab59b7a5a339ffc612ebd14051732d74b5b

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.12.1
+ - Log bulk request response body on error, not just when debug logging is enabled [#1096](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1096)
+
+## 11.12.0
+ - Add legacy template API support for Elasticsearch 8 [#1092](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1092)
+
 ## 11.11.0
  - When using an `api_key` along with either `cloud_id` or https `hosts`, you no longer need to also specify `ssl => true` [#1065](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1065)
 

data/docs/index.asciidoc

@@ -362,6 +362,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-template_api>> |<<string,string>>, one of `["auto", "legacy", "composable"]`|No
 | <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
@@ -1080,6 +1081,22 @@ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.
 You can set the path to your own template here, if you so desire.
 If not set, the included template will be used.
 
+[id="plugins-{type}s-{plugin}-template_api"]
+===== `template_api`
+
+  * Value can be any of: `auto`, `legacy`, `composable`
+  * Default value is `auto`
+
+The default setting of `auto` will use
+{ref}/index-templates.html[index template API] to create index template, if the
+Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher,
+and use {ref}/indices-templates-v1.html[legacy template API] otherwise.
+
+Setting this flag to `legacy` will use legacy template API to create index template.
+Setting this flag to `composable` will use index template API to create index template.
+
+NOTE: The format of template provided to <<plugins-{type}s-{plugin}-template>> needs to match the template API being used.
+
 [id="plugins-{type}s-{plugin}-template_name"]
 ===== `template_name`
 

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -77,12 +77,12 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def template_install(name, template, force=false)
-      if template_exists?(name) && !force
+    def template_install(template_endpoint, name, template, force=false)
+      if template_exists?(template_endpoint, name) && !force
         @logger.debug("Found existing Elasticsearch template, skipping template management", name: name)
         return
       end
-      template_put(name, template)
+      template_put(template_endpoint, name, template)
     end
 
     def last_es_version
@@ -402,20 +402,16 @@ module LogStash; module Outputs; class ElasticSearch;
       response.code >= 200 && response.code <= 299
     end
 
-    def template_exists?(name)
+    def template_exists?(template_endpoint, name)
       exists?("/#{template_endpoint}/#{name}")
     end
 
-    def template_put(name, template)
+    def template_put(template_endpoint, name, template)
       path = "#{template_endpoint}/#{name}"
       logger.info("Installing Elasticsearch template", name: name)
       @pool.put(path, nil, LogStash::Json.dump(template))
     end
 
-    def template_endpoint
-      maximum_seen_major_version < 8 ? '_template' : '_index_template'
-    end
-
     # ILM methods
 
     # check whether rollover alias already exists

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -1,8 +1,20 @@
 module LogStash; module Outputs; class ElasticSearch
   class TemplateManager
+    LEGACY_TEMPLATE_ENDPOINT = '_template'.freeze
+    INDEX_TEMPLATE_ENDPOINT = '_index_template'.freeze
+
     # To be mixed into the elasticsearch plugin base
     def self.install_template(plugin)
       return unless plugin.manage_template
+
+      if plugin.maximum_seen_major_version < 8 && plugin.template_api == 'auto'
+        plugin.logger.warn("`template_api => auto` resolved to `legacy` since we are connected to " + "Elasticsearch #{plugin.maximum_seen_major_version}, " +
+                           "but will resolve to `composable` the first time it connects to Elasticsearch 8+. " +
+                           "We recommend either setting `template_api => legacy` to continue providing legacy-style templates, " +
+                           "or migrating your template to the composable style and setting `template_api => composable`. " +
+                           "The legacy template API is slated for removal in Elasticsearch 9.")
+      end
+
       if plugin.template
         plugin.logger.info("Using mapping template from", :path => plugin.template)
         template = read_template_file(plugin.template)
@@ -14,7 +26,7 @@ module LogStash; module Outputs; class ElasticSearch
 
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
       plugin.logger.debug("Attempting to install template", template: template)
-      install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
+      install(plugin.client, template_endpoint(plugin), template_name(plugin), template, plugin.template_overwrite)
     end
 
     private
@@ -25,8 +37,8 @@ module LogStash; module Outputs; class ElasticSearch
       fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
     end
 
-    def self.install(client, template_name, template, template_overwrite)
-      client.template_install(template_name, template, template_overwrite)
+    def self.install(client, template_endpoint, template_name, template, template_overwrite)
+      client.template_install(template_endpoint, template_name, template, template_overwrite)
     end
 
     def self.add_ilm_settings_to_template(plugin, template)
@@ -63,5 +75,16 @@ module LogStash; module Outputs; class ElasticSearch
       template_data = ::IO.read(template_path)
       LogStash::Json.load(template_data)
     end
+
+    def self.template_endpoint(plugin)
+      if plugin.template_api == 'auto'
+        plugin.maximum_seen_major_version < 8 ? LEGACY_TEMPLATE_ENDPOINT : INDEX_TEMPLATE_ENDPOINT
+      elsif plugin.template_api.to_s == 'legacy'
+        LEGACY_TEMPLATE_ENDPOINT
+      else
+        INDEX_TEMPLATE_ENDPOINT
+      end
+    end
+
   end
 end end end

data/lib/logstash/outputs/elasticsearch.rb

@@ -185,6 +185,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # the "logstash" template (i.e. removing all customized settings)
   config :template_overwrite, :validate => :boolean, :default => false
 
+  # Flag for enabling legacy template api for Elasticsearch 8
+  # Default auto will use index template api for Elasticsearch 8 and use legacy api for 7
+  # Set to legacy to use legacy template api
+  config :template_api, :validate => ['auto', 'legacy', 'composable'], :default => 'auto'
+
   # The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
   # See https://www.elastic.co/blog/elasticsearch-versioning-support.
   config :version, :validate => :string

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -335,8 +335,8 @@ module LogStash; module PluginMixins; module ElasticSearch
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         @bulk_request_metrics.increment(:failures)
-        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s, :content_length => e.request_body.bytesize}
-        log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
+        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s,
+                    :content_length => e.request_body.bytesize, :body => e.response_body}
         message = "Encountered a retryable error (will retry with exponential backoff)"
 
         # We treat 429s as a special case because these really aren't errors, but

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.11.0'
+  s.version         = '11.12.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -140,27 +140,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     end
   end
 
-  describe "index template" do
-    subject { described_class.new(base_options) }
-    let(:template_name) { "logstash" }
-    let(:template) { {} }
-    let(:get_response) {
-      double("response", :body => {})
-    }
-
-    it "should call composable index template in version 8+" do
-      expect(subject).to receive(:maximum_seen_major_version).and_return(8)
-      expect(subject.pool).to receive(:put).with("_index_template/#{template_name}", nil, anything).and_return(get_response)
-      subject.template_put(template_name, template)
-    end
-
-    it "should call index template in version < 8" do
-      expect(subject).to receive(:maximum_seen_major_version).and_return(7)
-      expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
-      subject.template_put(template_name, template)
-    end
-  end
-
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -63,4 +63,53 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
       end
     end
   end
+
+  describe "template endpoint" do
+    describe "template_api => 'auto'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'auto'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use index template API" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
+        end
+      end
+
+      describe "in version < 8" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+
+    describe "template_api => 'legacy'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'legacy'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:maximum_seen_major_version).never
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+
+    describe "template_api => 'composable'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'composable'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:maximum_seen_major_version).never
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager:: INDEX_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.11.0
+  version: 11.12.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-12 00:00:00.000000000 Z
+date: 2022-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement