logstash-output-elasticsearch 9.2.4-java → 9.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 42465226c4184be1d0a210a70c4baf76219818de876bb05e7223c6e33575cbd3
-  data.tar.gz: 4f547ee1a60601b1412fb9adfbb53de1b07bfbc87215e871089c09e7e593dec1
+  metadata.gz: 5c6ed2c43c43667688e700d6f8db2b714b865a5653c0fc8892a8de479857397d
+  data.tar.gz: bf24c972d847db154b5feb1290341e92b53fdbf5673ea7916d651b77633f9e30
 SHA512:
-  metadata.gz: 2bf8b3f9ffe85bbd63e4456131271f06cad92acc95033cb79c40900b8199ee8795d8267aeb488085d65d0b07de9a9361710906573f76f818ed9f271acea90d06
-  data.tar.gz: ddac41655af9d2af7b1d2bf0dff94bcbb497daca068efc7423c3af1ca8cb14b235e8afedf81bc0864a968d1f685039b458c78e1af1b91114135e66e34e8312bf
+  metadata.gz: 51939e7522f9370efdc381bf5d981c0efb7a42aa9fc68a873c2281eba9bd4497ec63c40ac951c1b1f2c9c65398073632b13a9870dad05a6d15685dae7c9c6e38
+  data.tar.gz: 18b48cd4103103fcf957718afcaa5f24cfdf429e5ce5949a16a6674ea79035c4801f53049bda2d2a6693469abaf9bb5699547301705961d51159383b1c05a96a

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 9.3.0
+  - Adds support for Index Lifecycle Management for Elasticsearch 6.6.0 and above, running with at least a Basic License(Beta) [#805](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/805)
+
 ## 9.2.4
   - Fixed support for Elasticsearch 7.x [#812](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/812)
 

data/docs/index.asciidoc

@@ -75,13 +75,13 @@ HTTP requests to the bulk API are expected to return a 200 response code. All ot
 
 The following document errors are handled as follows:
 
-  * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<dlq-policy>> for more info.
+  * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<plugins-{type}s-{plugin}-dlq-policy>> for more info.
   * 409 errors (conflict) are logged as a warning and dropped. 
 
 Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
 It is more performant for Elasticsearch to retry these exceptions than this plugin.
 
-[[dlq-policy]]
+[id="plugins-{type}s-{plugin}-dlq-policy"]
 ==== DLQ Policy
 
 Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately
@@ -93,6 +93,56 @@ re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error
 happens, the problem is logged as a warning, and the event is dropped. See
 <<dead-letter-queues>> for more information about processing events in the DLQ.
 
+[id="plugins-{type}s-{plugin}-ilm"]
+==== Index Lifecycle Management (Beta)
+
+beta[]
+
+[NOTE]
+This feature requires an Elasticsearch instance of 6.6.0 or higher with at least a Basic license
+
+Logstash can use the {ref}/index-lifecycle-management.html[Index Lifecycle Management] to automate the management of indices over time.
+
+To configure the Elasticsearch output to use Index Lifecycle Management, set the `ilm_enabled` flag to true in the output definition:
+
+[source,ruby]
+    output {
+      elasticsearch {
+        ilm_enabled => true
+      }
+    }
+
+This will overwrite the index settings and adjust the Logstash template to write the necessary settings for the template
+to support index lifecycle management, including the index policy and rollover alias to be used.
+
+Logstash will create a rollover alias for the indices to be written to, including a pattern for how the actual indices will be named, and unless an ILM policy that already exists has been specified,
+a default policy will also be created. The default policy is configured to rollover an index when it reaches either 50 gigabytes in size, or is 30 days old, whichever happens first.
+
+The default rollover alias is called `logstash`, with a default pattern for the rollover index of `{now/d}-00001`,
+which will name indices on the date that the index is rolled over, followed by an incrementing number. Note that the pattern must end with a dash and a number that will be incremented.
+
+See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover API documentation] for more details on naming.
+
+The rollover alias, ilm pattern and policy can be modified.
+
+See config below for an example:
+[source,ruby]
+    output {
+      elasticsearch {
+        ilm_enabled => true
+        ilm_rollover_alias: "custom"
+        ilm_pattern: "000001"
+        ilm_policy: "custom_policy"
+      }
+    }
+
+NOTE: Custom ILM policies must already exist on the Elasticsearch cluster before they can be used.
+
+NOTE: If the rollover alias or pattern is modified, the index template will need to be overwritten as the settings `index.lifecycle.name` and `index.lifecycle.rollover_alias` are automatically written to the template
+
+NOTE: If the index property is supplied in the output definition, it will be overwritten by the rollover alias.
+
+
 ==== Batch Sizes
 
 This plugin attempts to send batches of events as a single request. However, if
@@ -139,6 +189,10 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ilm_enabled>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ilm_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ilm_policy>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ilm_rollover_alias>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
@@ -314,6 +368,54 @@ Any special characters present in the URLs here MUST be URL escaped! This means
 
 Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
 
+[id="plugins-{type}s-{plugin}-ilm_enabled"]
+===== `ilm_enabled`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Setting this flag to `true` will enable indices to be managed by the Index Lifecycle Management feature in Elasticsearch.
+
+NOTE: This feature requires a Basic License or above to be installed on an Elasticsearch cluster version 6.6.0 or later
+
+[id="plugins-{type}s-{plugin}-ilm_pattern"]
+===== `ilm_pattern`
+
+  * Value type is <<string,string>>
+  * Default value is `{now/d}-000001`
+
+Pattern used for generating indices managed by Index Lifecycle Management. The value specified in the pattern will be appended to
+the write alias, and incremented automatically when a new index is created by ILM.
+
+Date Math can be used when specifying an ilm pattern, see {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover API docs] for details
+
+NOTE: Updating the pattern will require the index template to be rewritten
+
+NOTE: The pattern must finish with a dash and a number that will be automatically incremented when indices rollover.
+
+[id="plugins-{type}s-{plugin}-ilm_policy"]
+===== `ilm_policy`
+
+  * Value type is <<string,string>>
+  * Default value is `logstash`
+
+Modify this setting to use a custom Index Lifecycle Management policy, rather than the default. If this value is not set, the default policy will
+be automatically installed into Elasticsearch
+
+NOTE: If this setting is specified, the policy must already exist in Elasticsearch cluster.
+
+[id="plugins-{type}s-{plugin}-ilm_rollover_alias"]
+===== `ilm_rollover_alias`
+
+  * Value type is <<string,string>>
+  * Default value is `logstash`
+
+The rollover alias is the alias where indices managed using Index Lifecycle Management will be written to.
+
+NOTE: If both `index` and `ilm_rollover_alias` are specified, `ilm_rollover_alias` takes precedence.
+
+NOTE: Updating the rollover alias will require the index template to be rewritten
+
 [id="plugins-{type}s-{plugin}-index"]
 ===== `index` 
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -90,6 +90,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/outputs/elasticsearch/http_client_builder"
   require "logstash/outputs/elasticsearch/common_configs"
   require "logstash/outputs/elasticsearch/common"
+  require "logstash/outputs/elasticsearch/ilm"
 
   # Protocol agnostic (i.e. non-http, non-java specific) configs go here
   include(LogStash::Outputs::ElasticSearch::CommonConfigs)
@@ -97,6 +98,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Protocol agnostic methods
   include(LogStash::Outputs::ElasticSearch::Common)
 
+  # Methods for ILM support
+  include(LogStash::Outputs::ElasticSearch::Ilm)
+
   config_name "elasticsearch"
 
   # The Elasticsearch action to perform. Valid actions are:

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -23,10 +23,10 @@ module LogStash; module Outputs; class ElasticSearch;
 
       setup_hosts # properly sets @hosts
       build_client
+      setup_after_successful_connection
       check_action_validity
       @bulk_request_metrics = metric.namespace(:bulk_requests)
       @document_level_metrics = metric.namespace(:documents)
-      install_template_after_successful_connection
       @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
     end
 
@@ -38,7 +38,7 @@ module LogStash; module Outputs; class ElasticSearch;
       retrying_submit(events.map {|e| event_action_tuple(e)})
     end
 
-    def install_template_after_successful_connection
+    def setup_after_successful_connection
       @template_installer ||= Thread.new do
         sleep_interval = @retry_initial_interval
         until successful_connection? || @stopping.true?
@@ -46,7 +46,11 @@ module LogStash; module Outputs; class ElasticSearch;
           Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
           sleep_interval = next_sleep_interval(sleep_interval)
         end
-        install_template if successful_connection?
+        if successful_connection?
+          verify_ilm_readiness if ilm_enabled?
+          install_template
+          setup_ilm if ilm_enabled?
+        end
       end
     end
 
@@ -114,7 +118,6 @@ module LogStash; module Outputs; class ElasticSearch;
       client.maximum_seen_major_version
     end
 
-
     def routing_field_name
       maximum_seen_major_version >= 6 ? :routing : :_routing
     end
@@ -353,4 +356,4 @@ module LogStash; module Outputs; class ElasticSearch;
         !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
     end
   end
-end; end; end
+end end end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -2,6 +2,10 @@ require 'forwardable' # Needed for logstash core SafeURI. We need to patch this
 
 module LogStash; module Outputs; class ElasticSearch
   module CommonConfigs
+
+    DEFAULT_INDEX_NAME = "logstash-%{+YYYY.MM.dd}"
+    DEFAULT_POLICY = "logstash-policy"
+
     def self.included(mod)
       # The index to write events to. This can be dynamic using the `%{foo}` syntax.
       # The default value will partition your indices by day so you can more easily
@@ -10,7 +14,7 @@ module LogStash; module Outputs; class ElasticSearch
       # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
       # LS uses Joda to format the index pattern from event timestamp.
       # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
-      mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
+      mod.config :index, :validate => :string, :default => DEFAULT_INDEX_NAME
 
       mod.config :document_type, 
         :validate => :string, 
@@ -136,6 +140,24 @@ module LogStash; module Outputs; class ElasticSearch
       # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
       # here like `pipeline => "%{INGEST_PIPELINE}"`
       mod.config :pipeline, :validate => :string, :default => nil
+
+
+      # -----
+      # ILM configurations (beta)
+      # -----
+      # Flag for enabling Index Lifecycle Management integration.
+      mod.config :ilm_enabled, :validate => :boolean, :default => false
+
+      # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
+      mod.config :ilm_rollover_alias, :validate => :string, :default => 'logstash'
+
+      # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
+      # {now/d} is date math, and will insert the appropriate value automatically.
+      mod.config :ilm_pattern, :validate => :string, :default => '{now/d}-000001'
+
+      # ILM policy to use, if undefined the default policy will be used.
+      mod.config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
+
     end
   end
 end end end

data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json

@@ -0,0 +1,14 @@
+{
+  "policy" : {
+    "phases": {
+      "hot" : {
+        "actions" : {
+          "rollover" : {
+            "max_size" : "50gb",
+            "max_age":"30d"
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -337,17 +337,62 @@ module LogStash; module Outputs; class ElasticSearch;
       ::LogStash::Util::SafeURI.new(raw_url)
     end
 
-    def template_exists?(name)
-      response = @pool.head("/_template/#{name}")
+    def exists?(path, use_get=false)
+      response = use_get ? @pool.get(path) : @pool.head(path)
       response.code >= 200 && response.code <= 299
     end
 
+    def template_exists?(name)
+      exists?("/_template/#{name}")
+    end
+
     def template_put(name, template)
       path = "_template/#{name}"
       logger.info("Installing elasticsearch template to #{path}")
       @pool.put(path, nil, LogStash::Json.dump(template))
     end
 
+    # ILM methods
+
+    # check whether rollover alias already exists
+    def rollover_alias_exists?(name)
+      exists?(name)
+    end
+
+    # Create a new rollover alias
+    def rollover_alias_put(alias_name, alias_definition)
+      logger.info("Creating rollover alias #{alias_name}")
+      begin
+        @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+        # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        if e.response_code == 400
+            logger.info("Rollover Alias #{alias_name} already exists. Skipping")
+            return
+        end
+        raise e
+      end
+    end
+
+    def get_xpack_info
+      get("/_xpack")
+    end
+
+    def get_ilm_endpoint
+      @pool.get("/_ilm/policy")
+    end
+
+    def ilm_policy_exists?(name)
+      exists?("/_ilm/policy/#{name}", true)
+    end
+
+    def ilm_policy_put(name, policy)
+      path = "_ilm/policy/#{name}"
+      logger.info("Installing ILM policy #{policy} to #{path}")
+      @pool.put(path, nil, LogStash::Json.dump(policy))
+    end
+
+
     # Build a bulk item for an elasticsearch update action
     def update_action_builder(args, source)
       if args[:_script]

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -103,9 +103,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
       
       request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
-      
-      request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.path}".gsub(/\/{2,}/, "/")
-        
+
+      request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.raw_path}".gsub(/\/{2,}/, "/")
+
       request_uri
     end
 

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -0,0 +1,84 @@
+module LogStash; module Outputs; class ElasticSearch
+  module Ilm
+
+    ILM_POLICY_PATH = "default-ilm-policy.json"
+
+    def setup_ilm
+      return unless ilm_enabled?
+      @logger.info("Using Index lifecycle management - this feature is currently in beta.")
+      @logger.warn "Overwriting supplied index name with rollover alias #{@ilm_rollover_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME
+      @index = ilm_rollover_alias
+
+      maybe_create_rollover_alias
+      maybe_create_ilm_policy
+    end
+
+    def ilm_enabled?
+      @ilm_enabled
+    end
+
+    def verify_ilm_readiness
+      return unless ilm_enabled?
+
+      # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature
+      # available and enabled.
+      begin
+        xpack = client.get_xpack_info
+        features = xpack["features"]
+        ilm = features.nil? ? nil : features["ilm"]
+        raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" if features.nil? || ilm.nil?
+        raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not available in your Elasticsearch cluster" unless ilm['available']
+        raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not enabled in your Elasticsearch cluster" unless ilm['enabled']
+
+        unless ilm_policy_default? || client.ilm_policy_exists?(ilm_policy)
+          raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance"
+        end
+
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible
+        if e.response_code == 404
+          raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster"
+        elsif e.response_code == 400
+          raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster"
+        else
+          raise e
+        end
+      end
+    end
+
+    private
+
+    def ilm_policy_default?
+      ilm_policy == LogStash::Outputs::ElasticSearch::DEFAULT_POLICY
+    end
+
+    def maybe_create_ilm_policy
+      if ilm_policy_default? && !client.ilm_policy_exists?(ilm_policy)
+        client.ilm_policy_put(ilm_policy, policy_payload)
+      end
+    end
+
+    def maybe_create_rollover_alias
+      client.rollover_alias_put(rollover_alias_target, rollover_alias_payload) unless client.rollover_alias_exists?(ilm_rollover_alias)
+    end
+
+    def rollover_alias_target
+      "<#{ilm_rollover_alias}-#{ilm_pattern}>"
+    end
+
+    def rollover_alias_payload
+      {
+          'aliases' => {
+              ilm_rollover_alias =>{
+                  'is_write_index' =>  true
+              }
+          }
+      }
+    end
+
+    def policy_payload
+      policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__))
+      LogStash::Json.load(::IO.read(policy_path))
+    end
+  end
+ end end end