logstash-output-elasticsearch 11.15.9-java → 11.17.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9e537b9f31644ce80834b295b99d22566863f666ab319efc34f641c15018d74
-  data.tar.gz: a99f63dd55f4b0a12e597e812db124dd9a7fe82ce1a5e7af4057992f903eac65
+  metadata.gz: 37124c3a166313a2fb9f3831273def114770178158439a44ebfa1bc1d32f9d0f
+  data.tar.gz: a09fd3ce2c54908fedc14dc2d780bb3ec48d7091d05e2e8fcb5789e4ec1e30b9
 SHA512:
-  metadata.gz: 12fa3b203130210b5d274364ff97e31bfb01aaedd23ac22fc53fea1626cad628d3f33e952dcf12555fc4860d7577235684e255550dfc7668d9dc93d7e6bf55ff
-  data.tar.gz: 50ca989af2afc85f439995c6dde9c7eeda56924c9d9729ef91426b34dc99146fadecf3e290dc7e122113bb2cbf50bdc1eeac22f8448d2d4373b0d251660fb6a7
+  metadata.gz: caac996badd1bbdeb231fad3f40f96a50386baf78ee356587d5fc5d2b4a095f1073bee417a99aab081c13cb1a18802785abed618dc85db504f56145c620c46b6
+  data.tar.gz: 697a89b810998154a44338e8e73f02351b72afa00fc35e1fc115a22ce5ecfeddd27d29bd8fdf4fb779a3b17fbaa2f4f361a2c30b68c0b5ce0cd49a6edeba1a1d

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.17.0
+  - Added support to http compression level. Deprecated `http_compression` in favour of `compression_level` and enabled compression level 1 by default. [#1148](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1148)
+
+## 11.16.0
+  - Added support to Serverless Elasticsearch [#1445](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1145)
+
 ## 11.15.9
   - allow dlq_ settings when using data streams [#1144](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1144)
 

data/docs/index.asciidoc

@@ -277,9 +277,9 @@ not reevaluate its DNS value while the keepalive is in effect.
 ==== HTTP Compression
 
 This plugin always reads compressed responses from {es}.
-It _can be configured_ to send compressed bulk requests to {es}.
+By default, it sends compressed bulk requests to {es}.
 
-If you are concerned about bandwidth, you can enable <<plugins-{type}s-{plugin}-http_compression>> to trade a small amount of CPU capacity for a significant reduction in network IO.
+If you are concerned about bandwidth, you can set a higher <<plugins-{type}s-{plugin}-compression_level>> to trade CPU capacity for a reduction in network IO.
 
 ==== Authentication
 
@@ -310,6 +310,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-compression_level>> |<<number,number>>, one of `[0 ~ 9]`|No
 | <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-data_stream>> |<<string,string>>, one of `["true", "false", "auto"]`|No
 | <<plugins-{type}s-{plugin}-data_stream_auto_routing>> |<<boolean,boolean>>|No
@@ -459,6 +460,17 @@ Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 For more details, check out the
 {logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
 
+[id="plugins-{type}s-{plugin}-compression_level"]
+===== `compression_level`
+
+  * Value can be any of: `0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9`
+  * Default value is `1`
+
+The gzip compression level. Setting this value to `0` disables compression.
+The compression level must be in the range of `1` (best speed) to `9` (best compression).
+
+Increasing the compression level will reduce the network usage but will increase the CPU usage.
+
 [id="plugins-{type}s-{plugin}-data_stream"]
 ===== `data_stream`
 
@@ -618,7 +630,7 @@ NOTE: Deprecated, refer to <<plugins-{type}s-{plugin}-silence_errors_in_log>>.
 Pass a set of key value pairs as the headers sent in each request to
 an elasticsearch node. The headers will be used for any kind of request
 (_bulk request, template installation, health checks and sniffing).
-These custom headers will be overidden by settings like `http_compression`.
+These custom headers will be overidden by settings like `compression_level`.
 
 [id="plugins-{type}s-{plugin}-healthcheck_path"]
 ===== `healthcheck_path`
@@ -659,11 +671,12 @@ Any special characters present in the URLs here MUST be URL escaped! This means
 
 [id="plugins-{type}s-{plugin}-http_compression"]
 ===== `http_compression`
+deprecated[11.17.0, Replaced by <<plugins-{type}s-{plugin}-compression_level>>]
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-Enable gzip compression on requests.
+Setting `true` enables gzip compression level 1 on requests.
 
 This setting allows you to reduce this plugin's outbound network traffic by
 compressing each bulk _request_ to {es}.

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -48,6 +48,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       :sniffer_delay => 10,
     }.freeze
 
+    BUILD_FLAVOUR_SERVERLESS = 'serverless'.freeze
+
     def initialize(logger, adapter, initial_urls=[], options={})
       @logger = logger
       @adapter = adapter
@@ -75,6 +77,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @license_checker = options[:license_checker] || LogStash::PluginMixins::ElasticSearch::NoopLicenseChecker::INSTANCE
 
       @last_es_version = Concurrent::AtomicReference.new
+      @build_flavour = Concurrent::AtomicReference.new
     end
 
     def start
@@ -250,7 +253,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
           # If no exception was raised it must have succeeded!
           logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
           # We reconnected to this node, check its ES version
-          es_version = get_es_version(url)
+          version_info = get_es_version(url)
+          es_version = version_info.fetch('number', nil)
+          build_flavour = version_info.fetch('build_flavor', nil)
+
           if es_version.nil?
             logger.warn("Failed to retrieve Elasticsearch version data from connected endpoint, connection aborted", :url => url.sanitized.to_s)
             next
@@ -258,6 +264,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
           @state_mutex.synchronize do
             meta[:version] = es_version
             set_last_es_version(es_version, url)
+            set_build_flavour(build_flavour)
 
             alive = @license_checker.appropriate_license?(self, url)
             meta[:state] = alive ? :alive : :dead
@@ -475,7 +482,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       response = LogStash::Json.load(response.body)
 
-      response.fetch('version', {}).fetch('number', nil)
+      response.fetch('version', {})
     end
 
     def last_es_version
@@ -486,6 +493,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @state_mutex.synchronize { @maximum_seen_major_version }
     end
 
+    def serverless?
+      @build_flavour.get == BUILD_FLAVOUR_SERVERLESS
+    end
+
     private
 
     # @private executing within @state_mutex
@@ -515,5 +526,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
                    previous_major: @maximum_seen_major_version, new_major: major, node_url: url.sanitized.to_s)
     end
 
+    def set_build_flavour(flavour)
+      @build_flavour.set(flavour)
+    end
+
   end
 end; end; end; end;

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -93,6 +93,10 @@ module LogStash; module Outputs; class ElasticSearch;
       @pool.maximum_seen_major_version
     end
 
+    def serverless?
+      @pool.serverless?
+    end
+
     def alive_urls_count
       @pool.alive_urls_count
     end
@@ -114,7 +118,7 @@ module LogStash; module Outputs; class ElasticSearch;
       end
 
       body_stream = StringIO.new
-      if http_compression
+      if compression_level?
         body_stream.set_encoding "BINARY"
         stream_writer = gzip_writer(body_stream)
       else
@@ -137,14 +141,14 @@ module LogStash; module Outputs; class ElasticSearch;
                        :batch_offset => (index + 1 - batch_actions.size))
           bulk_responses << bulk_send(body_stream, batch_actions)
           body_stream.truncate(0) && body_stream.seek(0)
-          stream_writer = gzip_writer(body_stream) if http_compression
+          stream_writer = gzip_writer(body_stream) if compression_level?
           batch_actions.clear
         end
         stream_writer.write(as_json)
         batch_actions << action
       end
 
-      stream_writer.close if http_compression
+      stream_writer.close if compression_level?
 
       logger.debug("Sending final bulk request for batch.",
                    :action_count => batch_actions.size,
@@ -153,7 +157,7 @@ module LogStash; module Outputs; class ElasticSearch;
                    :batch_offset => (actions.size - batch_actions.size))
       bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
 
-      body_stream.close if !http_compression
+      body_stream.close unless compression_level?
       join_bulk_responses(bulk_responses)
     end
 
@@ -161,7 +165,7 @@ module LogStash; module Outputs; class ElasticSearch;
       fail(ArgumentError, "Cannot create gzip writer on IO with unread bytes") unless io.eof?
       fail(ArgumentError, "Cannot create gzip writer on non-empty IO") unless io.pos == 0
 
-      Zlib::GzipWriter.new(io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+      Zlib::GzipWriter.new(io, client_settings.fetch(:compression_level), Zlib::DEFAULT_STRATEGY)
     end
 
     def join_bulk_responses(bulk_responses)
@@ -172,7 +176,7 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def bulk_send(body_stream, batch_actions)
-      params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
+      params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {}
       response = @pool.post(@bulk_path, params, body_stream.string)
 
       @bulk_response_metrics.increment(response.code.to_s)
@@ -294,8 +298,10 @@ module LogStash; module Outputs; class ElasticSearch;
       @_ssl_options ||= client_settings.fetch(:ssl, {})
     end
 
-    def http_compression
-      client_settings.fetch(:http_compression, false)
+    # return true if compression_level is [1..9]
+    # return false if it is 0
+    def compression_level?
+      client_settings.fetch(:compression_level) > 0
     end
 
     def build_adapter(options)

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -8,7 +8,7 @@ module LogStash; module Outputs; class ElasticSearch;
         :pool_max => params["pool_max"],
         :pool_max_per_route => params["pool_max_per_route"],
         :check_connection_timeout => params["validate_after_inactivity"],
-        :http_compression => params["http_compression"],
+        :compression_level => params["compression_level"],
         :headers => params["custom_headers"] || {}
       }
       

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -14,11 +14,18 @@ module LogStash; module Outputs; class ElasticSearch
       return @ilm_actually_enabled if defined?(@ilm_actually_enabled)
       @ilm_actually_enabled =
         begin
-          if @ilm_enabled == 'auto'
+          if serverless?
+            raise LogStash::ConfigurationError, "Invalid ILM configuration `ilm_enabled => true`. " +
+              "Serverless Elasticsearch cluster does not support Index Lifecycle Management." if @ilm_enabled.to_s == 'true'
+            @logger.info("ILM auto configuration (`ilm_enabled => auto` or unset) resolved to `false`. "\
+              "Serverless Elasticsearch cluster does not support Index Lifecycle Management.") if @ilm_enabled == 'auto'
+            false
+          elsif @ilm_enabled == 'auto'
             if ilm_on_by_default?
               ilm_alias_set?
             else
-              @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - Your Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
+              @logger.info("ILM auto configuration (`ilm_enabled => auto` or unset) resolved to `false`."\
+                " Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
               false
             end
           elsif @ilm_enabled.to_s == 'true'

data/lib/logstash/outputs/elasticsearch/license_checker.rb

@@ -11,6 +11,8 @@ module LogStash; module Outputs; class ElasticSearch
     # @param url [LogStash::Util::SafeURI] ES node URL
     # @return [Boolean] true if provided license is deemed appropriate
     def appropriate_license?(pool, url)
+      return true if pool.serverless?
+
       license = extract_license(pool.get_license(url))
       case license_status(license)
       when 'active'

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -13,8 +13,11 @@ module LogStash; module Outputs; class ElasticSearch
                            "We recommend either setting `template_api => legacy` to continue providing legacy-style templates, " +
                            "or migrating your template to the composable style and setting `template_api => composable`. " +
                            "The legacy template API is slated for removal in Elasticsearch 9.")
+      elsif plugin.template_api == 'legacy' && plugin.serverless?
+        raise LogStash::ConfigurationError, "Invalid template configuration `template_api => legacy`. Serverless Elasticsearch does not support legacy template API."
       end
 
+
       if plugin.template
         plugin.logger.info("Using mapping template from", :path => plugin.template)
         template = read_template_file(plugin.template)
@@ -61,11 +64,13 @@ module LogStash; module Outputs; class ElasticSearch
         plugin.logger.trace("Resolving ILM template settings: under 'settings' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version)
         legacy_index_template_settings(template)
       else
-        template_endpoint = template_endpoint(plugin)
-        plugin.logger.trace("Resolving ILM template settings: template doesn't have 'settings' or 'template' fields, falling back to auto detection", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version, :template_endpoint => template_endpoint)
-        template_endpoint == INDEX_TEMPLATE_ENDPOINT ?
-          composable_index_template_settings(template) :
+        use_index_template_api = index_template_api?(plugin)
+        plugin.logger.trace("Resolving ILM template settings: template doesn't have 'settings' or 'template' fields, falling back to auto detection", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version, :index_template_api => use_index_template_api)
+        if use_index_template_api
+          composable_index_template_settings(template)
+        else
           legacy_index_template_settings(template)
+        end
       end
     end
 
@@ -100,12 +105,25 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.template_endpoint(plugin)
-      if plugin.template_api == 'auto'
-        plugin.maximum_seen_major_version < 8 ? LEGACY_TEMPLATE_ENDPOINT : INDEX_TEMPLATE_ENDPOINT
-      elsif plugin.template_api.to_s == 'legacy'
-        LEGACY_TEMPLATE_ENDPOINT
+      index_template_api?(plugin) ? INDEX_TEMPLATE_ENDPOINT : LEGACY_TEMPLATE_ENDPOINT
+    end
+
+    def self.index_template_api?(plugin)
+      case plugin.serverless?
+      when true
+        true
       else
-        INDEX_TEMPLATE_ENDPOINT
+        case plugin.template_api
+        when 'auto'
+          plugin.maximum_seen_major_version >= 8
+        when 'composable'
+          true
+        when 'legacy'
+          false
+        else
+          plugin.logger.warn("Invalid template_api value #{plugin.template_api}")
+          true
+        end
       end
     end
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -276,6 +276,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     super
     setup_ecs_compatibility_related_defaults
     setup_ssl_params!
+    setup_compression_level!
   end
 
   def register
@@ -368,6 +369,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
         params['proxy'] = proxy # do not do resolving again
       end
     end
+
     super(params)
   end
 
@@ -669,6 +671,20 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
   end
 
+  def setup_compression_level!
+    @compression_level = normalize_config(:compression_level) do |normalize|
+      normalize.with_deprecated_mapping(:http_compression) do |http_compression|
+        if http_compression == true
+          DEFAULT_ZIP_LEVEL
+        else
+          0
+        end
+      end
+    end
+
+    params['compression_level'] = @compression_level unless @compression_level.nil?
+  end
+
   # To be overidden by the -java version
   VALID_HTTP_ACTIONS = ["index", "delete", "create", "update"]
   def valid_actions

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -7,6 +7,7 @@ module LogStash; module PluginMixins; module ElasticSearch
     # This module defines common options that can be reused by alternate elasticsearch output plugins such as the elasticsearch_data_streams output.
 
     DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//127.0.0.1")
+    DEFAULT_ZIP_LEVEL = 1
 
     CONFIG_PARAMS = {
         # Username to authenticate to a secure Elasticsearch cluster
@@ -186,7 +187,14 @@ module LogStash; module PluginMixins; module ElasticSearch
         :validate_after_inactivity => { :validate => :number, :default => 10000 },
 
         # Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
-        :http_compression => { :validate => :boolean, :default => false },
+        # Set `true` to enable compression with level 1
+        # Set `false` to disable compression with level 0
+        :http_compression => { :validate => :boolean, :default => true, :deprecated => "Set 'compression_level' instead." },
+
+        # Number `1` ~ `9` are the gzip compression level
+        # Set `0` to disable compression
+        # Set `1` (best speed) to `9` (best compression) to use compression
+        :compression_level => { :validate => [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ], :default => DEFAULT_ZIP_LEVEL },
 
         # Custom Headers to send on each request to elasticsearch nodes
         :custom_headers => { :validate => :hash, :default => {} },

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -145,6 +145,10 @@ module LogStash; module PluginMixins; module ElasticSearch
       client.maximum_seen_major_version
     end
 
+    def serverless?
+      client.serverless?
+    end
+
     def alive_urls_count
       client.alive_urls_count
     end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.15.9'
+  s.version         = '11.17.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -59,11 +59,14 @@ module ESHelper
   end
 
   def self.es_version
-    [
-      nilify(RSpec.configuration.filter[:es_version]),
-      nilify(ENV['ES_VERSION']),
-      nilify(ENV['ELASTIC_STACK_VERSION']),
-    ].compact.first
+    {
+      "number" => [
+        nilify(RSpec.configuration.filter[:es_version]),
+        nilify(ENV['ES_VERSION']),
+        nilify(ENV['ELASTIC_STACK_VERSION']),
+      ].compact.first,
+      "build_flavor" => 'default'
+    }
   end
 
   RSpec::Matchers.define :have_hits do |expected|

data/spec/fixtures/license_check/active.json

@@ -0,0 +1,16 @@
+{
+  "license": {
+    "status": "active",
+    "uid": "d85d2c6a-b96d-3cc6-96db-5571a789b156",
+    "type": "enterprise",
+    "issue_date": "1970-01-01T00:00:00.000Z",
+    "issue_date_in_millis": 0,
+    "expiry_date": "2100-01-01T00:00:00.000Z",
+    "expiry_date_in_millis": 4102444800000,
+    "max_nodes": null,
+    "max_resource_units": 100000,
+    "issued_to": "Elastic Cloud",
+    "issuer": "API",
+    "start_date_in_millis": 0
+  }
+}

data/spec/fixtures/license_check/inactive.json

@@ -0,0 +1,5 @@
+{
+  "license": {
+    "status": "inactive"
+  }
+}

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -8,63 +8,64 @@ RSpec::Matchers.define :a_valid_gzip_encoded_string do
   }
 end
 
-describe "indexing with http_compression turned on", :integration => true do
-  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
-  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
-  let(:event_count) { 10000 + rand(500) }
-  let(:events) { event_count.times.map { event }.to_a }
-  let(:config) {
-    {
-      "hosts" => get_host_port,
-      "index" => index,
-      "http_compression" => true
+[ {"http_compression" => true}, {"compression_level" => 1} ].each do |compression_config|
+  describe "indexing with http_compression turned on", :integration => true do
+    let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+    let(:event_count) { 10000 + rand(500) }
+    let(:events) { event_count.times.map { event }.to_a }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index
+      }
     }
-  }
-  subject { LogStash::Outputs::ElasticSearch.new(config) }
+    subject { LogStash::Outputs::ElasticSearch.new(config.merge(compression_config)) }
 
-  let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
-  let(:http_client_options) { {} }
-  let(:http_client) do
-    Manticore::Client.new(http_client_options)
-  end
+    let(:es_url) { "http://#{get_host_port}" }
+    let(:index_url) {"#{es_url}/#{index}"}
+    let(:http_client_options) { {} }
+    let(:http_client) do
+      Manticore::Client.new(http_client_options)
+    end
 
-  before do
-    subject.register
-    subject.multi_receive([])
-  end
+    before do
+      subject.register
+      subject.multi_receive([])
+    end
 
-  shared_examples "an indexer" do
-    it "ships events" do
-      subject.multi_receive(events)
+    shared_examples "an indexer" do
+      it "ships events" do
+        subject.multi_receive(events)
 
-      http_client.post("#{es_url}/_refresh").call
+        http_client.post("#{es_url}/_refresh").call
 
-      response = http_client.get("#{index_url}/_count?q=*")
-      result = LogStash::Json.load(response.body)
-      cur_count = result["count"]
-      expect(cur_count).to eq(event_count)
+        response = http_client.get("#{index_url}/_count?q=*")
+        result = LogStash::Json.load(response.body)
+        cur_count = result["count"]
+        expect(cur_count).to eq(event_count)
 
-      response = http_client.get("#{index_url}/_search?q=*&size=1000")
-      result = LogStash::Json.load(response.body)
-      result["hits"]["hits"].each do |doc|
-        if ESHelper.es_version_satisfies?("< 8")
-          expect(doc["_type"]).to eq(type)
-        else
-          expect(doc).not_to include("_type")
+        response = http_client.get("#{index_url}/_search?q=*&size=1000")
+        result = LogStash::Json.load(response.body)
+        result["hits"]["hits"].each do |doc|
+          if ESHelper.es_version_satisfies?("< 8")
+            expect(doc["_type"]).to eq(type)
+          else
+            expect(doc).not_to include("_type")
+          end
+          expect(doc["_index"]).to eq(index)
         end
-        expect(doc["_index"]).to eq(index)
       end
     end
-  end
 
-  it "sets the correct content-encoding header and body is compressed" do
-    expect(subject.client.pool.adapter.client).to receive(:send).
-      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
-      and_call_original
-    subject.multi_receive(events)
-  end
+    it "sets the correct content-encoding header and body is compressed" do
+      expect(subject.client.pool.adapter.client).to receive(:send).
+        with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+        and_call_original
+      subject.multi_receive(events)
+    end
 
-  it_behaves_like("an indexer")
-end
+    it_behaves_like("an indexer")
+  end
+end

data/spec/integration/outputs/index_spec.rb

@@ -262,7 +262,8 @@ describe "indexing" do
     let(:config) {
       {
         "hosts" => get_host_port,
-        "index" => index
+        "index" => index,
+        "http_compression" => false
       }
     }
     it_behaves_like("an indexer")
@@ -273,7 +274,8 @@ describe "indexing" do
     let(:config) {
       {
         "hosts" => get_host_port,
-        "index" => index
+        "index" => index,
+        "http_compression" => false
       }
     }
     it_behaves_like("an indexer")
@@ -291,7 +293,8 @@ describe "indexing" do
         "password" => password,
         "ssl_enabled" => true,
         "ssl_certificate_authorities" => cacert,
-        "index" => index
+        "index" => index,
+        "http_compression" => false
       }
     end 
 
@@ -351,7 +354,8 @@ describe "indexing" do
               "hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
               "ssl_enabled" => true,
               "ssl_certificate_authorities" => "spec/fixtures/test_certs/test.crt",
-              "index" => index
+              "index" => index,
+              "http_compression" => false
           }
         end
 

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -6,8 +6,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
   let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
-  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
-  let(:es_node_versions) { [ "0.0.0" ] }
+  let(:options) { {:resurrect_delay => 3, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
+  let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'} ] }
   let(:license_status) { 'active' }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
@@ -22,7 +22,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
-    allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
+    allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_version_info)
     allow(subject.license_checker).to receive(:license_status).and_return(license_status)
   end
 
@@ -267,13 +267,37 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     context "if there are nodes with multiple major versions" do
-      let(:es_node_versions) { [ "0.0.0", "6.0.0" ] }
+      let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'}, { "number" => '6.0.0', "build_flavor" => 'default'} ] }
       it "picks the largest major version" do
         expect(subject.maximum_seen_major_version).to eq(6)
       end
     end
   end
 
+
+  describe "build flavour tracking" do
+    let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://somehost:9200")] }
+
+    let(:es_version_info) { [ { "number" => '8.9.0', "build_flavor" => "serverless" } ] }
+
+    let(:valid_response) { MockResponse.new(200,
+                                            {"tagline" => "You Know, for Search",
+                                                  "version" => {
+                                                    "number" => '8.9.0',
+                                                    "build_flavor" => LogStash::Outputs::ElasticSearch::HttpClient::Pool::BUILD_FLAVOUR_SERVERLESS} },
+                                            { "X-Elastic-Product" => "Elasticsearch" }
+    ) }
+
+    before(:each) do
+      allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
+      subject.start
+    end
+
+    it "picks the build flavour" do
+      expect(subject.serverless?).to be_truthy
+    end
+  end
+
   describe "license checking" do
     before(:each) do
       allow(subject).to receive(:health_check_request)
@@ -364,7 +388,7 @@ describe "#elasticsearch?" do
   let(:adapter) { double("Manticore Adapter") }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
-  let(:es_node_versions) { [ "0.0.0" ] }
+  let(:es_version_info) { [{ "number" => '0.0.0', "build_flavor" => 'default'}] }
   let(:license_status) { 'active' }
 
   subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -183,6 +183,25 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     end
   end
 
+  describe "compression_level?" do
+    subject { described_class.new(base_options) }
+    let(:base_options) { super().merge(:client_settings => {:compression_level => compression_level}) }
+
+    context "with client_settings `compression_level => 1`" do
+      let(:compression_level) { 1 }
+      it "gives true" do
+        expect(subject.compression_level?).to be_truthy
+      end
+    end
+
+    context "with client_settings `compression_level => 0`" do
+      let(:compression_level) { 0 }
+      it "gives false" do
+        expect(subject.compression_level?).to be_falsey
+      end
+    end
+  end
+
   describe "#bulk" do
     subject(:http_client) { described_class.new(base_options) }
 
@@ -192,13 +211,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
     ]}
 
-    [true,false].each do |http_compression_enabled|
-      context "with `http_compression => #{http_compression_enabled}`" do
+    [0, 9].each do |compression_level|
+      context "with `compression_level => #{compression_level}`" do
 
-        let(:base_options) { super().merge(:client_settings => {:http_compression => http_compression_enabled}) }
+        let(:base_options) { super().merge(:client_settings => {:compression_level => compression_level}) }
+        let(:compression_level_enabled) { compression_level > 0 }
 
         before(:each) do
-          if http_compression_enabled
+          if compression_level_enabled
             expect(http_client).to receive(:gzip_writer).at_least(:once).and_call_original
           else
             expect(http_client).to_not receive(:gzip_writer)
@@ -212,7 +232,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
           it "should be handled properly" do
             allow(subject).to receive(:join_bulk_responses)
             expect(subject).to receive(:bulk_send).once do |data|
-              if !http_compression_enabled
+              if !compression_level_enabled
                 expect(data.size).to be > target_bulk_bytes
               else
                 expect(Zlib::gunzip(data.string).size).to be > target_bulk_bytes

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -73,6 +73,7 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
         let(:template_api) { "composable" }
 
         it 'resolves composable index template API compatible setting' do
+          expect(plugin).to receive(:serverless?).and_return(false)
           expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log
           template = {}
           described_class.resolve_template_settings(plugin, template)
@@ -84,6 +85,7 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
         let(:template_api) { "legacy" }
 
         it 'resolves legacy index template API compatible setting' do
+          expect(plugin).to receive(:serverless?).and_return(false)
           expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log
           template = {}
           described_class.resolve_template_settings(plugin, template)
@@ -97,6 +99,7 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
         describe "with ES < 8 versions" do
 
           it 'resolves legacy index template API compatible setting' do
+            expect(plugin).to receive(:serverless?).and_return(false)
             expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
             template = {}
             described_class.resolve_template_settings(plugin, template)
@@ -106,6 +109,7 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
         describe "with ES >= 8 versions" do
           it 'resolves composable index template API compatible setting' do
+            expect(plugin).to receive(:serverless?).and_return(false)
             expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
             template = {}
             described_class.resolve_template_settings(plugin, template)
@@ -123,6 +127,7 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
       describe "in version 8+" do
         it "should use index template API" do
+          expect(plugin).to receive(:serverless?).and_return(false)
           expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
           endpoint = described_class.template_endpoint(plugin)
           expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
@@ -131,6 +136,7 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
       describe "in version < 8" do
         it "should use legacy template API" do
+          expect(plugin).to receive(:serverless?).and_return(false)
           expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
           endpoint = described_class.template_endpoint(plugin)
           expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
@@ -144,6 +150,7 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
       describe "in version 8+" do
         it "should use legacy template API" do
+          expect(plugin).to receive(:serverless?).and_return(false)
           expect(plugin).to receive(:maximum_seen_major_version).never
           endpoint = described_class.template_endpoint(plugin)
           expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
@@ -157,11 +164,26 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
       describe "in version 8+" do
         it "should use legacy template API" do
+          expect(plugin).to receive(:serverless?).and_return(false)
           expect(plugin).to receive(:maximum_seen_major_version).never
           endpoint = described_class.template_endpoint(plugin)
-          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager:: INDEX_TEMPLATE_ENDPOINT)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
         end
       end
     end
+
+    describe "in serverless" do
+      [:auto, :composable, :legacy].each do |api|
+        let(:plugin_settings) { {"manage_template" => true, "template_api" => api.to_s} }
+        let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+        it "use index template API when template_api set to #{api}" do
+          expect(plugin).to receive(:serverless?).and_return(true)
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
+        end
+      end
+
+    end
   end
 end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -474,7 +474,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     context "unexpected bulk response" do
       let(:options) do
-        { "hosts" => "127.0.0.1:9999", "index" => "%{foo}", "manage_template" => false }
+        { "hosts" => "127.0.0.1:9999", "index" => "%{foo}", "manage_template" => false, "http_compression" => false }
       end
 
       let(:events) { [ ::LogStash::Event.new("foo" => "bar1"), ::LogStash::Event.new("foo" => "bar2") ] }
@@ -624,6 +624,7 @@ describe LogStash::Outputs::ElasticSearch do
   end
 
   context '413 errors' do
+    let(:options) { super().merge("http_compression" => "false") }
     let(:payload_size) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES + 1024 }
     let(:event) { ::LogStash::Event.new("message" => ("a" * payload_size ) ) }
 
@@ -1557,6 +1558,37 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "http compression" do
+    describe "initialize setting" do
+      context "with `http_compression` => true" do
+        let(:options) { super().merge('http_compression' => true) }
+        it "set compression level to 1" do
+          subject.register
+          expect(subject.instance_variable_get(:@compression_level)).to eq(1)
+        end
+      end
+
+      context "with `http_compression` => false" do
+        let(:options) { super().merge('http_compression' => false) }
+        it "set compression level to 0" do
+          subject.register
+          expect(subject.instance_variable_get(:@compression_level)).to eq(0)
+        end
+      end
+
+      [0, 9].each do |config|
+        context "with `compression_level` => #{config}" do
+          let(:options) { super().merge('compression_level' => config) }
+          it "keeps the setting" do
+            subject.register
+            expect(subject.instance_variable_get(:@compression_level)).to eq(config)
+          end
+        end
+      end
+    end
+
+  end
+
   @private
 
   def stub_manticore_client!(manticore_double = nil)

data/spec/unit/outputs/license_check_spec.rb

@@ -21,5 +21,37 @@ describe LogStash::Outputs::ElasticSearch::LicenseChecker do
       expect(LogStash::Outputs::ElasticSearch::HttpClient::Pool.instance_methods).to include(:get_license)
     end
   end
+
+  context "appropriate license" do
+    let(:logger) { double("logger") }
+    let(:url) { LogStash::Util::SafeURI.new("https://cloud.elastic.co") }
+    let(:pool) { double("pool") }
+    subject { described_class.new(logger) }
+
+    it "is true when connect to serverless" do
+      allow(pool).to receive(:serverless?).and_return(true)
+      expect(subject.appropriate_license?(pool, url)).to eq true
+    end
+
+    it "is true when license status is active" do
+      allow(pool).to receive(:serverless?).and_return(false)
+      allow(pool).to receive(:get_license).with(url).and_return(LogStash::Json.load File.read("spec/fixtures/license_check/active.json"))
+      expect(subject.appropriate_license?(pool, url)).to eq true
+    end
+
+    it "is true when license status is inactive" do
+      allow(logger).to receive(:warn).with(instance_of(String), anything)
+      allow(pool).to receive(:serverless?).and_return(false)
+      allow(pool).to receive(:get_license).with(url).and_return(LogStash::Json.load File.read("spec/fixtures/license_check/inactive.json"))
+      expect(subject.appropriate_license?(pool, url)).to eq true
+    end
+
+    it "is false when no license return" do
+      allow(logger).to receive(:error).with(instance_of(String), anything)
+      allow(pool).to receive(:serverless?).and_return(false)
+      allow(pool).to receive(:get_license).with(url).and_return(LogStash::Json.load('{}'))
+      expect(subject.appropriate_license?(pool, url)).to eq false
+    end
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.15.9
+  version: 11.17.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-18 00:00:00.000000000 Z
+date: 2023-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -279,6 +279,8 @@ files:
 - spec/fixtures/_nodes/6x.json
 - spec/fixtures/_nodes/7x.json
 - spec/fixtures/htpasswd
+- spec/fixtures/license_check/active.json
+- spec/fixtures/license_check/inactive.json
 - spec/fixtures/nginx_reverse_proxy.conf
 - spec/fixtures/scripts/painless/scripted_update.painless
 - spec/fixtures/scripts/painless/scripted_update_nested.painless
@@ -365,6 +367,8 @@ test_files:
 - spec/fixtures/_nodes/6x.json
 - spec/fixtures/_nodes/7x.json
 - spec/fixtures/htpasswd
+- spec/fixtures/license_check/active.json
+- spec/fixtures/license_check/inactive.json
 - spec/fixtures/nginx_reverse_proxy.conf
 - spec/fixtures/scripts/painless/scripted_update.painless
 - spec/fixtures/scripts/painless/scripted_update_nested.painless