logstash-output-elasticsearch 10.8.1-java → 11.0.0-java

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -1,6 +1,4 @@
 require "logstash/outputs/elasticsearch"
-require "cabin"
-require "base64"
 require 'logstash/outputs/elasticsearch/http_client/pool'
 require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
 require 'cgi'
@@ -80,12 +78,16 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def template_install(name, template, force=false)
       if template_exists?(name) && !force
-        @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
+        @logger.debug("Found existing Elasticsearch template, skipping template management", name: name)
         return
       end
       template_put(name, template)
     end
 
+    def last_es_version
+      @pool.last_es_version
+    end
+
     def maximum_seen_major_version
       @pool.maximum_seen_major_version
     end
@@ -109,27 +111,50 @@ module LogStash; module Outputs; class ElasticSearch;
       body_stream = StringIO.new
       if http_compression
         body_stream.set_encoding "BINARY"
-        stream_writer = Zlib::GzipWriter.new(body_stream, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
-      else 
+        stream_writer = gzip_writer(body_stream)
+      else
         stream_writer = body_stream
       end
       bulk_responses = []
-      bulk_actions.each do |action|
+      batch_actions = []
+      bulk_actions.each_with_index do |action, index|
         as_json = action.is_a?(Array) ?
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-        if (body_stream.size + as_json.bytesize) > TARGET_BULK_BYTES
-          bulk_responses << bulk_send(body_stream) unless body_stream.size == 0
+        if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
+          stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
+          logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
+                       :action_count => batch_actions.size,
+                       :payload_size => stream_writer.pos,
+                       :content_length => body_stream.size,
+                       :batch_offset => (index + 1 - batch_actions.size))
+          bulk_responses << bulk_send(body_stream, batch_actions)
+          body_stream.truncate(0) && body_stream.seek(0)
+          stream_writer = gzip_writer(body_stream) if http_compression
+          batch_actions.clear
         end
         stream_writer.write(as_json)
+        batch_actions << action
       end
       stream_writer.close if http_compression
-      bulk_responses << bulk_send(body_stream) if body_stream.size > 0
+      logger.debug("Sending final bulk request for batch.",
+                   :action_count => batch_actions.size,
+                   :payload_size => stream_writer.pos,
+                   :content_length => body_stream.size,
+                   :batch_offset => (actions.size - batch_actions.size))
+      bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
       body_stream.close if !http_compression
       join_bulk_responses(bulk_responses)
     end
 
+    def gzip_writer(io)
+      fail(ArgumentError, "Cannot create gzip writer on IO with unread bytes") unless io.eof?
+      fail(ArgumentError, "Cannot create gzip writer on non-empty IO") unless io.pos == 0
+
+      Zlib::GzipWriter.new(io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+    end
+
     def join_bulk_responses(bulk_responses)
       {
         "errors" => bulk_responses.any? {|r| r["errors"] == true},
@@ -137,25 +162,37 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def bulk_send(body_stream)
+    def bulk_send(body_stream, batch_actions)
       params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
-      # Discard the URL
       response = @pool.post(@bulk_path, params, body_stream.string)
-      if !body_stream.closed?
-        body_stream.truncate(0)
-        body_stream.seek(0)
-      end
 
       @bulk_response_metrics.increment(response.code.to_s)
 
-      if response.code != 200
+      case response.code
+      when 200 # OK
+        LogStash::Json.load(response.body)
+      when 413 # Payload Too Large
+        logger.warn("Bulk request rejected: `413 Payload Too Large`", :action_count => batch_actions.size, :content_length => body_stream.size)
+        emulate_batch_error_response(batch_actions, response.code, 'payload_too_large')
+      else
         url = ::LogStash::Util::SafeURI.new(response.final_url)
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
           response.code, url, body_stream.to_s, response.body
         )
       end
+    end
 
-      LogStash::Json.load(response.body)
+    def emulate_batch_error_response(actions, http_code, reason)
+      {
+          "errors" => true,
+          "items" => actions.map do |action|
+            action = action.first if action.is_a?(Array)
+            request_action, request_parameters = action.first
+            {
+                request_action => {"status" => http_code, "error" => { "type" => reason }}
+            }
+          end
+      }
     end
 
     def get(path)
@@ -349,7 +386,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def template_put(name, template)
       path = "#{template_endpoint}/#{name}"
-      logger.info("Installing elasticsearch template to #{path}")
+      logger.info("Installing Elasticsearch template", name: name)
       @pool.put(path, nil, LogStash::Json.dump(template))
     end
 
@@ -366,13 +403,13 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Create a new rollover alias
     def rollover_alias_put(alias_name, alias_definition)
-      logger.info("Creating rollover alias #{alias_name}")
       begin
         @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+        logger.info("Created rollover alias", name: alias_name)
         # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         if e.response_code == 400
-            logger.info("Rollover Alias #{alias_name} already exists. Skipping")
+            logger.info("Rollover alias already exists, skipping", name: alias_name)
             return
         end
         raise e
@@ -393,13 +430,14 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def ilm_policy_put(name, policy)
       path = "_ilm/policy/#{name}"
-      logger.info("Installing ILM policy #{policy} to #{path}")
+      logger.info("Installing ILM policy #{policy}", name: name)
       @pool.put(path, nil, LogStash::Json.dump(policy))
     end
 
 
     # Build a bulk item for an elasticsearch update action
     def update_action_builder(args, source)
+      args = args.clone()
       if args[:_script]
         # Use the event as a hash from your script with variable name defined
         # by script_var_name (default: "event")

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -1,3 +1,4 @@
+require "concurrent/atomic/atomic_reference"
 require "logstash/plugin_mixins/elasticsearch/noop_license_checker"
 
 module LogStash; module Outputs; class ElasticSearch; class HttpClient;
@@ -71,6 +72,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @stopping = false
 
       @license_checker = options[:license_checker] || LogStash::PluginMixins::ElasticSearch::NoopLicenseChecker::INSTANCE
+
+      @last_es_version = Concurrent::AtomicReference.new
     end
 
     def start
@@ -118,12 +121,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @state_mutex.synchronize { @url_info }
     end
 
-    def maximum_seen_major_version
-      @state_mutex.synchronize do
-        @maximum_seen_major_version
-      end
-    end
-
     def urls
       url_info.keys
     end
@@ -252,11 +249,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       response = perform_request_to_url(url, :get, LICENSE_PATH)
       LogStash::Json.load(response.body)
     rescue => e
-      logger.error("Unable to get license information", url: url.sanitized.to_s, error_type: e.class, error: e.message)
+      logger.error("Unable to get license information", url: url.sanitized.to_s, exception: e.class, message: e.message)
       {}
     end
 
     def health_check_request(url)
+      logger.debug("Running health check to see if an ES connection is working", url: url.sanitized.to_s, path: @healthcheck_path)
       perform_request_to_url(url, :head, @healthcheck_path)
     end
 
@@ -264,29 +262,20 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       # Try to keep locking granularity low such that we don't affect IO...
       @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
         begin
-          logger.debug("Running health check to see if an Elasticsearch connection is working",
-                        :healthcheck_url => url, :path => @healthcheck_path)
           health_check_request(url)
           # If no exception was raised it must have succeeded!
-          logger.warn("Restored connection to ES instance", :url => url.sanitized.to_s)
+          logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
           # We reconnected to this node, check its ES version
           es_version = get_es_version(url)
           @state_mutex.synchronize do
             meta[:version] = es_version
-            major = major_version(es_version)
-            if !@maximum_seen_major_version
-              @logger.info("ES Output version determined", :es_version => major)
-              set_new_major_version(major)
-            elsif major > @maximum_seen_major_version
-              @logger.warn("Detected a node with a higher major version than previously observed. This could be the result of an elasticsearch cluster upgrade.", :previous_major => @maximum_seen_major_version, :new_major => major, :node_url => url.sanitized.to_s)
-              set_new_major_version(major)
-            end
+            set_last_es_version(es_version, url)
 
             alive = @license_checker.appropriate_license?(self, url)
             meta[:state] = alive ? :alive : :dead
           end
         rescue HostUnreachableError, BadResponseCodeError => e
-          logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: url.sanitized.to_s, error_type: e.class, error: e.message)
+          logger.warn("Attempted to resurrect connection to dead ES instance, but got an error", url: url.sanitized.to_s, exception: e.class, message: e.message)
         end
       end
     end
@@ -355,9 +344,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
 
       if state_changes[:removed].size > 0 || state_changes[:added].size > 0
-        if logger.info?
-          logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
-        end
+        logger.info? && logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
       end
       
       # Run an inline healthcheck anytime URLs are updated
@@ -371,10 +358,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @state_mutex.synchronize { @url_info.size }
     end
 
-    def es_versions
-      @state_mutex.synchronize { @url_info.size }
-    end
-
     def add_url(url)
       @url_info[url] ||= empty_url_meta
     end
@@ -459,22 +442,52 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
     def return_connection(url)
       @state_mutex.synchronize do
-        if @url_info[url] # Guard against the condition where the connection has already been deleted
-          @url_info[url][:in_use] -= 1
-        end
+        info = @url_info[url]
+        info[:in_use] -= 1 if info # Guard against the condition where the connection has already been deleted
       end
     end
 
     def get_es_version(url)
       request = perform_request_to_url(url, :get, ROOT_URI_PATH)
-      LogStash::Json.load(request.body)["version"]["number"]
+      LogStash::Json.load(request.body)["version"]["number"] # e.g. "7.10.0"
+    end
+
+    def last_es_version
+      @last_es_version.get
+    end
+
+    def maximum_seen_major_version
+      @state_mutex.synchronize { @maximum_seen_major_version }
+    end
+
+    private
+
+    # @private executing within @state_mutex
+    def set_last_es_version(version, url)
+      @last_es_version.set(version)
+
+      major = major_version(version)
+      if @maximum_seen_major_version.nil?
+        @logger.info("Elasticsearch version determined (#{version})", es_version: major)
+        set_maximum_seen_major_version(major)
+      elsif major > @maximum_seen_major_version
+        warn_on_higher_major_version(major, url)
+        @maximum_seen_major_version = major
+      end
     end
 
-    def set_new_major_version(version)
-      @maximum_seen_major_version = version
-      if @maximum_seen_major_version >= 6
-        @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", :es_version => @maximum_seen_major_version)
+    def set_maximum_seen_major_version(major)
+      if major >= 6
+        @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", es_version: major)
       end
+      @maximum_seen_major_version = major
     end
+
+    def warn_on_higher_major_version(major, url)
+      @logger.warn("Detected a node with a higher major version than previously observed, " +
+                   "this could be the result of an Elasticsearch cluster upgrade",
+                   previous_major: @maximum_seen_major_version, new_major: major, node_url: url.sanitized.to_s)
+    end
+
   end
 end; end; end; end;

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -4,21 +4,12 @@ module LogStash; module Outputs; class ElasticSearch
     ILM_POLICY_PATH = "default-ilm-policy.json"
 
     def setup_ilm
-      return unless ilm_in_use?
       logger.warn("Overwriting supplied index #{@index} with rollover alias #{@ilm_rollover_alias}") unless default_index?(@index)
       @index = @ilm_rollover_alias
       maybe_create_rollover_alias
       maybe_create_ilm_policy
     end
 
-    def default_rollover_alias?(rollover_alias)
-      rollover_alias == default_ilm_rollover_alias
-    end
-
-    def ilm_alias_set?
-      default_index?(@index) || !default_rollover_alias?(@ilm_rollover_alias)
-    end
-
     def ilm_in_use?
       return @ilm_actually_enabled if defined?(@ilm_actually_enabled)
       @ilm_actually_enabled =
@@ -46,6 +37,12 @@ module LogStash; module Outputs; class ElasticSearch
         end
     end
 
+    private
+
+    def ilm_alias_set?
+      default_index?(@index) || !default_rollover_alias?(@ilm_rollover_alias)
+    end
+
     def ilm_on_by_default?
       maximum_seen_major_version >= 7
     end
@@ -73,19 +70,21 @@ module LogStash; module Outputs; class ElasticSearch
       end
     end
 
-    private
-
     def default_index?(index)
       index == @default_index
     end
 
+    def default_rollover_alias?(rollover_alias)
+      rollover_alias == default_ilm_rollover_alias
+    end
+
     def ilm_policy_default?
       ilm_policy == LogStash::Outputs::ElasticSearch::DEFAULT_POLICY
     end
 
     def maybe_create_ilm_policy
       if ilm_policy_default?
-          client.ilm_policy_put(ilm_policy, policy_payload) unless client.ilm_policy_exists?(ilm_policy)
+        client.ilm_policy_put(ilm_policy, policy_payload) unless client.ilm_policy_exists?(ilm_policy)
       else
         raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance" unless client.ilm_policy_exists?(ilm_policy)
       end

data/lib/logstash/outputs/elasticsearch/license_checker.rb

@@ -7,41 +7,38 @@ module LogStash; module Outputs; class ElasticSearch
 
     # Figure out if the provided license is appropriate or not
     # The appropriate_license? methods is the method called from LogStash::Outputs::ElasticSearch::HttpClient::Pool#healthcheck!
+    # @param pool
     # @param url [LogStash::Util::SafeURI] ES node URL
-    # @param license [Hash] ES node deserialized licence document
     # @return [Boolean] true if provided license is deemed appropriate
     def appropriate_license?(pool, url)
-      return true if oss?
-
       license = pool.get_license(url)
-      if valid_es_license?(license)
+      case license_status(license)
+      when 'active'
         true
-      else
-        # As this version is to be shipped with Logstash 7.x we won't mark the connection as unlicensed
-        #
-        #  @logger.error("Cannot connect to the Elasticsearch cluster configured in the Elasticsearch output. Logstash requires the default distribution of Elasticsearch. Please update to the default distribution of Elasticsearch for full access to all free features, or switch to the OSS distribution of Logstash.", :url => url.sanitized.to_s)
-        #  meta[:state] = :unlicensed
-        #
-        # Instead we'll log a deprecation warning and mark it as alive:
-        #
-        log_license_deprecation_warn(url)
+      when nil
+        warn_no_license(url)
+        false
+      else # 'invalid', 'expired'
+        warn_invalid_license(url, license)
         true
       end
     end
 
-    # Note that oss? could be private but is used by the Pool specs
-    def oss?
-      LogStash::OSS
+    def license_status(license)
+      license.fetch("license", {}).fetch("status", nil)
     end
 
-    # Note that valid_es_license? could be private but is used by the Pool specs
-    def valid_es_license?(license)
-      license.fetch("license", {}).fetch("status", nil) == "active"
+    private
+
+    def warn_no_license(url)
+      @logger.error("Connecting to an OSS distribution of Elasticsearch is no longer supported, " +
+                    "please upgrade to the default distribution of Elasticsearch", url: url.sanitized.to_s)
     end
 
-    # Note that log_license_deprecation_warn could be private but is used by the Pool specs
-    def log_license_deprecation_warn(url)
-      @logger.warn("DEPRECATION WARNING: Connecting to an OSS distribution of Elasticsearch using the default distribution of Logstash will stop working in Logstash 8.0.0. Please upgrade to the default distribution of Elasticsearch, or use the OSS distribution of Logstash", :url => url.sanitized.to_s)
+    def warn_invalid_license(url, license)
+      @logger.warn("WARNING: Current Elasticsearch license is not active, " +
+                   "please check Elasticsearch's licensing information", url: url.sanitized.to_s, license: license)
     end
+
   end
 end; end; end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -13,10 +13,8 @@ module LogStash; module Outputs; class ElasticSearch
       end
 
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
-      plugin.logger.info("Attempting to install template", :manage_template => template)
+      plugin.logger.debug("Attempting to install template", template: template)
       install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
-    rescue => e
-      plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
     end
 
     private
@@ -38,7 +36,7 @@ module LogStash; module Outputs; class ElasticSearch
       template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
       settings = template_settings(plugin, template)
       if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias'])
-        plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.")
+        plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled")
       end
       settings.update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
     end
@@ -61,7 +59,7 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.read_template_file(template_path)
-      raise ArgumentError, "Template file '#{template_path}' could not be found!" unless ::File.exists?(template_path)
+      raise ArgumentError, "Template file '#{template_path}' could not be found" unless ::File.exists?(template_path)
       template_data = ::IO.read(template_path)
       LogStash::Json.load(template_data)
     end