logstash-output-elasticsearch-test 10.3.0-x86_64-linux

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -0,0 +1,441 @@
+require "logstash/outputs/elasticsearch/template_manager"
+
+module LogStash; module Outputs; class ElasticSearch;
+  module Common
+    attr_reader :client, :hosts
+
+    # These codes apply to documents, not at the request level
+    DOC_DLQ_CODES = [400, 404]
+    DOC_SUCCESS_CODES = [200, 201]
+    DOC_CONFLICT_CODE = 409
+
+    # When you use external versioning, you are communicating that you want
+    # to ignore conflicts. More obviously, since an external version is a
+    # constant part of the incoming document, we should not retry, as retrying
+    # will never succeed.
+    VERSION_TYPES_PERMITTING_CONFLICT = ["external", "external_gt", "external_gte"]
+
+    def register
+      @template_installed = Concurrent::AtomicBoolean.new(false)
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
+      @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
+
+      fill_hosts_from_cloud_id
+      fill_user_password_from_cloud_auth
+      setup_hosts # properly sets @hosts
+      build_client
+      setup_after_successful_connection
+      check_action_validity
+      @bulk_request_metrics = metric.namespace(:bulk_requests)
+      @document_level_metrics = metric.namespace(:documents)
+      @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
+    end
+
+    # Receive an array of events and immediately attempt to index them (no buffering)
+    def multi_receive(events)
+      until @template_installed.true?
+        sleep 1
+      end
+      retrying_submit(events.map {|e| event_action_tuple(e)})
+    end
+
+    def setup_after_successful_connection
+      @template_installer ||= Thread.new do
+        sleep_interval = @retry_initial_interval
+        until successful_connection? || @stopping.true?
+          @logger.debug("Waiting for connectivity to Elasticsearch cluster. Retrying in #{sleep_interval}s")
+          Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+          sleep_interval = next_sleep_interval(sleep_interval)
+        end
+        if successful_connection?
+          discover_cluster_uuid
+          install_template
+          setup_ilm if ilm_in_use?
+        end
+      end
+    end
+
+    def stop_template_installer
+      @template_installer.join unless @template_installer.nil?
+    end
+
+    def successful_connection?
+      !!maximum_seen_major_version
+    end
+
+    def use_event_type?(client)
+      client.maximum_seen_major_version < 8
+    end
+
+    # Convert the event into a 3-tuple of action, params, and event
+    def event_action_tuple(event)
+      action = event.sprintf(@action)
+
+      params = {
+        :_id => @document_id ? event.sprintf(@document_id) : nil,
+        :_index => event.sprintf(@index),
+        routing_field_name => @routing ? event.sprintf(@routing) : nil
+      }
+
+      params[:_type] = get_event_type(event) if use_event_type?(client)
+
+      if @pipeline
+        params[:pipeline] = event.sprintf(@pipeline)
+      end
+
+      if @parent
+        if @join_field
+          join_value = event.get(@join_field)
+          parent_value = event.sprintf(@parent)
+          event.set(@join_field, { "name" => join_value, "parent" => parent_value })
+          params[routing_field_name] = event.sprintf(@parent)
+        else
+          params[:parent] = event.sprintf(@parent)
+        end
+      end
+
+      if action == 'update'
+        params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
+        params[:_script] = event.sprintf(@script) if @script != ""
+        params[retry_on_conflict_action_name] = @retry_on_conflict
+      end
+
+      if @version
+        params[:version] = event.sprintf(@version)
+      end
+
+      if @version_type
+        params[:version_type] = event.sprintf(@version_type)
+      end
+
+      [action, params, event]
+    end
+
+    def setup_hosts
+      @hosts = Array(@hosts)
+      if @hosts.empty?
+        @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
+        @hosts.replace(["localhost"])
+      end
+    end
+
+    def hosts_default?(hosts)
+      # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
+      hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(CommonConfigs::DEFAULT_HOST)
+    end
+    private :hosts_default?
+
+    def fill_hosts_from_cloud_id
+      return unless @cloud_id
+
+      if @hosts && !hosts_default?(@hosts)
+        raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+      end
+      @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+    end
+
+    def fill_user_password_from_cloud_auth
+      return unless @cloud_auth
+
+      if @user || @password
+        raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
+      end
+      @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
+      params['user'], params['password'] = @user, @password
+    end
+
+    def parse_host_uri_from_cloud_id(cloud_id)
+      begin # might not be available on older LS
+        require 'logstash/util/cloud_setting_id'
+      rescue LoadError
+        raise LogStash::ConfigurationError, 'The cloud_id setting is not supported by your version of Logstash, ' +
+            'please upgrade your installation (or set hosts instead).'
+      end
+
+      begin
+        cloud_id = LogStash::Util::CloudSettingId.new(cloud_id) # already does append ':{port}' to host
+      rescue ArgumentError => e
+        raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Id/i, 'cloud_id')
+      end
+      cloud_uri = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
+      LogStash::Util::SafeURI.new(cloud_uri)
+    end
+    private :parse_host_uri_from_cloud_id
+
+    def parse_user_password_from_cloud_auth(cloud_auth)
+      begin # might not be available on older LS
+        require 'logstash/util/cloud_setting_auth'
+      rescue LoadError
+        raise LogStash::ConfigurationError, 'The cloud_auth setting is not supported by your version of Logstash, ' +
+            'please upgrade your installation (or set user/password instead).'
+      end
+
+      cloud_auth = cloud_auth.value if cloud_auth.is_a?(LogStash::Util::Password)
+      begin
+        cloud_auth = LogStash::Util::CloudSettingAuth.new(cloud_auth)
+      rescue ArgumentError => e
+        raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Auth/i, 'cloud_auth')
+      end
+      [ cloud_auth.username, cloud_auth.password ]
+    end
+    private :parse_user_password_from_cloud_auth
+
+    def maximum_seen_major_version
+      client.maximum_seen_major_version
+    end
+
+    def routing_field_name
+      maximum_seen_major_version >= 6 ? :routing : :_routing
+    end
+
+    def retry_on_conflict_action_name
+      maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
+    end
+
+    def install_template
+      TemplateManager.install_template(self)
+      @template_installed.make_true
+    end
+
+    def discover_cluster_uuid
+      return unless defined?(plugin_metadata)
+      cluster_info = client.get('/')
+      plugin_metadata.set(:cluster_uuid, cluster_info['cluster_uuid'])
+    rescue => e
+      # TODO introducing this logging message breaks many tests that need refactoring
+      # @logger.error("Unable to retrieve elasticsearch cluster uuid", error => e.message)
+    end
+
+    def check_action_validity
+      raise LogStash::ConfigurationError, "No action specified!" unless @action
+
+      # If we're using string interpolation, we're good!
+      return if @action =~ /%{.+}/
+      return if valid_actions.include?(@action)
+
+      raise LogStash::ConfigurationError, "Action '#{@action}' is invalid! Pick one of #{valid_actions} or use a sprintf style statement"
+    end
+
+    # To be overidden by the -java version
+    VALID_HTTP_ACTIONS=["index", "delete", "create", "update"]
+    def valid_actions
+      VALID_HTTP_ACTIONS
+    end
+
+    def retrying_submit(actions)
+      # Initially we submit the full list of actions
+      submit_actions = actions
+
+      sleep_interval = @retry_initial_interval
+
+      while submit_actions && submit_actions.length > 0
+
+        # We retry with whatever is didn't succeed
+        begin
+          submit_actions = submit(submit_actions)
+          if submit_actions && submit_actions.size > 0
+            @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request.", :count => submit_actions.size)
+          end
+        rescue => e
+          @logger.error("Encountered an unexpected error submitting a bulk request! Will retry.",
+                       :error_message => e.message,
+                       :class => e.class.name,
+                       :backtrace => e.backtrace)
+        end
+
+        # Everything was a success!
+        break if !submit_actions || submit_actions.empty?
+
+        # If we're retrying the action sleep for the recommended interval
+        # Double the interval for the next time through to achieve exponential backoff
+        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+        sleep_interval = next_sleep_interval(sleep_interval)
+      end
+    end
+
+    def sleep_for_interval(sleep_interval)
+      Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+      next_sleep_interval(sleep_interval)
+    end
+
+    def next_sleep_interval(current_interval)
+      doubled = current_interval * 2
+      doubled > @retry_max_interval ? @retry_max_interval : doubled
+    end
+
+    def submit(actions)
+      bulk_response = safe_bulk(actions)
+
+      # If the response is nil that means we were in a retry loop
+      # and aborted since we're shutting down
+      return if bulk_response.nil?
+
+      # If it did return and there are no errors we're good as well
+      if bulk_response["errors"]
+        @bulk_request_metrics.increment(:with_errors)
+      else
+        @bulk_request_metrics.increment(:successes)
+        @document_level_metrics.increment(:successes, actions.size)
+        return
+      end
+
+      actions_to_retry = []
+      bulk_response["items"].each_with_index do |response,idx|
+        action_type, action_props = response.first
+
+        status = action_props["status"]
+        failure  = action_props["error"]
+        action = actions[idx]
+        action_params = action[1]
+
+        # Retry logic: If it is success, we move on. If it is a failure, we have 3 paths:
+        # - For 409, we log and drop. there is nothing we can do
+        # - For a mapping error, we send to dead letter queue for a human to intervene at a later point.
+        # - For everything else there's mastercard. Yep, and we retry indefinitely. This should fix #572 and other transient network issues
+        if DOC_SUCCESS_CODES.include?(status)
+          @document_level_metrics.increment(:successes)
+          next
+        elsif DOC_CONFLICT_CODE == status
+          @document_level_metrics.increment(:non_retryable_failures)
+          @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
+          next
+        elsif DOC_DLQ_CODES.include?(status)
+          handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
+          @document_level_metrics.increment(:non_retryable_failures)
+          next
+        else
+          # only log what the user whitelisted
+          @document_level_metrics.increment(:retryable_failures)
+          @logger.info "retrying failed action with response code: #{status} (#{failure})" if !failure_type_logging_whitelist.include?(failure["type"])
+          actions_to_retry << action
+        end
+      end
+
+      actions_to_retry
+    end
+
+    def handle_dlq_status(message, action, status, response)
+      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
+      if @dlq_writer
+        # TODO: Change this to send a map with { :status => status, :action => action } in the future
+        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
+      else
+        error_type = response.fetch('index', {}).fetch('error', {})['type']
+        if 'invalid_index_name_exception' == error_type
+          level = :error
+        else
+          level = :warn
+        end
+        @logger.send level, message, status: status, action: action, response: response
+      end
+    end
+
+    # Determine the correct value for the 'type' field for the given event
+    DEFAULT_EVENT_TYPE_ES6="doc".freeze
+    DEFAULT_EVENT_TYPE_ES7="_doc".freeze
+    def get_event_type(event)
+      # Set the 'type' value for the index.
+      type = if @document_type
+               event.sprintf(@document_type)
+             else
+               if client.maximum_seen_major_version < 6
+                 event.get("type") || DEFAULT_EVENT_TYPE_ES6
+               elsif client.maximum_seen_major_version == 6
+                 DEFAULT_EVENT_TYPE_ES6
+               elsif client.maximum_seen_major_version == 7
+                 DEFAULT_EVENT_TYPE_ES7
+               else
+                 nil
+               end
+             end
+
+      if !(type.is_a?(String) || type.is_a?(Numeric))
+        @logger.warn("Bad event type! Non-string/integer type value set!", :type_class => type.class, :type_value => type.to_s, :event => event)
+      end
+
+      type.to_s
+    end
+
+    # Rescue retryable errors during bulk submission
+    def safe_bulk(actions)
+      sleep_interval = @retry_initial_interval
+      begin
+        es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]}
+        response = @client.bulk(es_actions)
+        response
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
+        # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
+        # and let the user sort it out from there
+        @logger.error(
+          "Attempted to send a bulk request to elasticsearch'"+
+            " but Elasticsearch appears to be unreachable or down!",
+          :error_message => e.message,
+          :class => e.class.name,
+          :will_retry_in_seconds => sleep_interval
+        )
+        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+
+        # We retry until there are no errors! Errors should all go to the retry queue
+        sleep_interval = sleep_for_interval(sleep_interval)
+        @bulk_request_metrics.increment(:failures)
+        retry unless @stopping.true?
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
+        @logger.error(
+          "Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?",
+          :error_message => e.message,
+          :class => e.class.name,
+          :will_retry_in_seconds => sleep_interval
+        )
+        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+        sleep_interval = next_sleep_interval(sleep_interval)
+        @bulk_request_metrics.increment(:failures)
+        retry unless @stopping.true?
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        @bulk_request_metrics.increment(:failures)
+        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
+        log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
+        message = "Encountered a retryable error. Will Retry with exponential backoff "
+
+        # We treat 429s as a special case because these really aren't errors, but
+        # rather just ES telling us to back off a bit, which we do.
+        # The other retryable code is 503, which are true errors
+        # Even though we retry the user should be made aware of these
+        if e.response_code == 429
+          logger.debug(message, log_hash)
+        else
+          logger.error(message, log_hash)
+        end
+
+        sleep_interval = sleep_for_interval(sleep_interval)
+        retry
+      rescue => e
+        # Stuff that should never happen
+        # For all other errors print out full connection issues
+        @logger.error(
+          "An unknown error occurred sending a bulk request to Elasticsearch. We will retry indefinitely",
+          :error_message => e.message,
+          :error_class => e.class.name,
+          :backtrace => e.backtrace
+        )
+
+        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+
+        sleep_interval = sleep_for_interval(sleep_interval)
+        @bulk_request_metrics.increment(:failures)
+        retry unless @stopping.true?
+      end
+    end
+
+    def default_index?(index)
+      @index == LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME
+    end
+
+    def dlq_enabled?
+      # TODO there should be a better way to query if DLQ is enabled
+      # See more in: https://github.com/elastic/logstash/issues/8064
+      respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
+        !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
+    end
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -0,0 +1,167 @@
+require 'forwardable' # Needed for logstash core SafeURI. We need to patch this in core: https://github.com/elastic/logstash/pull/5978
+
+module LogStash; module Outputs; class ElasticSearch
+  module CommonConfigs
+
+    DEFAULT_INDEX_NAME = "logstash-%{+YYYY.MM.dd}"
+    DEFAULT_POLICY = "logstash-policy"
+    DEFAULT_ROLLOVER_ALIAS = 'logstash'
+
+    DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//127.0.0.1")
+
+    def self.included(mod)
+      # The index to write events to. This can be dynamic using the `%{foo}` syntax.
+      # The default value will partition your indices by day so you can more easily
+      # delete old data or only search specific date ranges.
+      # Indexes may not contain uppercase characters.
+      # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
+      # LS uses Joda to format the index pattern from event timestamp.
+      # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
+      mod.config :index, :validate => :string, :default => DEFAULT_INDEX_NAME
+
+      mod.config :document_type, 
+        :validate => :string, 
+        :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
+
+      # From Logstash 1.3 onwards, a template is applied to Elasticsearch during
+      # Logstash's startup if one with the name `template_name` does not already exist.
+      # By default, the contents of this template is the default template for
+      # `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
+      # `logstash-*`.  Should you require support for other index names, or would like
+      # to change the mappings in the template in general, a custom template can be
+      # specified by setting `template` to the path of a template file.
+      #
+      # Setting `manage_template` to false disables this feature.  If you require more
+      # control over template creation, (e.g. creating indices dynamically based on
+      # field names) you should set `manage_template` to false and use the REST
+      # API to apply your templates manually.
+      mod.config :manage_template, :validate => :boolean, :default => true
+
+      # This configuration option defines how the template is named inside Elasticsearch.
+      # Note that if you have used the template management features and subsequently
+      # change this, you will need to prune the old template manually, e.g.
+      #
+      # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
+      #
+      # where `OldTemplateName` is whatever the former setting was.
+      mod.config :template_name, :validate => :string, :default => "logstash"
+
+      # You can set the path to your own template here, if you so desire.
+      # If not set, the included template will be used.
+      mod.config :template, :validate => :path
+
+      # The template_overwrite option will always overwrite the indicated template
+      # in Elasticsearch with either the one indicated by template or the included one.
+      # This option is set to false by default. If you always want to stay up to date
+      # with the template provided by Logstash, this option could be very useful to you.
+      # Likewise, if you have your own template file managed by puppet, for example, and
+      # you wanted to be able to update it regularly, this option could help there as well.
+      #
+      # Please note that if you are using your own customized version of the Logstash
+      # template (logstash), setting this to true will make Logstash to overwrite
+      # the "logstash" template (i.e. removing all customized settings)
+      mod.config :template_overwrite, :validate => :boolean, :default => false
+
+      # The document ID for the index. Useful for overwriting existing entries in
+      # Elasticsearch with the same ID.
+      mod.config :document_id, :validate => :string
+
+      # The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
+      # See https://www.elastic.co/blog/elasticsearch-versioning-support.
+      mod.config :version, :validate => :string
+      
+      # The version_type to use for indexing.
+      # See https://www.elastic.co/blog/elasticsearch-versioning-support.
+      # See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#_version_types
+      mod.config :version_type, :validate => ["internal", 'external', "external_gt", "external_gte", "force"]
+
+      # A routing override to be applied to all processed events.
+      # This can be dynamic using the `%{foo}` syntax.
+      mod.config :routing, :validate => :string
+
+      # For child documents, ID of the associated parent.
+      # This can be dynamic using the `%{foo}` syntax.
+      mod.config :parent, :validate => :string, :default => nil
+
+      # For child documents, name of the join field
+      mod.config :join_field, :validate => :string, :default => nil
+
+      # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
+      # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
+      #     `"127.0.0.1"`
+      #     `["127.0.0.1:9200","127.0.0.2:9200"]`
+      #     `["http://127.0.0.1"]`
+      #     `["https://127.0.0.1:9200"]`
+      #     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
+      # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
+      # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
+      #
+      # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
+      mod.config :hosts, :validate => :uri, :default => [ DEFAULT_HOST ], :list => true
+
+      # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+      #
+      # For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[cloud documentation]
+      mod.config :cloud_id, :validate => :string
+
+      # Set upsert content for update mode.s
+      # Create a new document with this parameter as json string if `document_id` doesn't exists
+      mod.config :upsert, :validate => :string, :default => ""
+
+      # Enable `doc_as_upsert` for update mode.
+      # Create a new document with source if `document_id` doesn't exist in Elasticsearch
+      mod.config :doc_as_upsert, :validate => :boolean, :default => false
+
+      # Set script name for scripted update mode
+      mod.config :script, :validate => :string, :default => ""
+
+      # Define the type of script referenced by "script" variable
+      #  inline : "script" contains inline script
+      #  indexed : "script" contains the name of script directly indexed in elasticsearch
+      #  file    : "script" contains the name of script stored in elasticseach's config directory
+      mod.config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"]
+
+      # Set the language of the used script. If not set, this defaults to painless in ES 5.0
+      mod.config :script_lang, :validate => :string, :default => "painless"
+
+      # Set variable name passed to script (scripted update)
+      mod.config :script_var_name, :validate => :string, :default => "event"
+
+      # if enabled, script is in charge of creating non-existent document (scripted update)
+      mod.config :scripted_upsert, :validate => :boolean, :default => false
+
+      # Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
+      mod.config :retry_initial_interval, :validate => :number, :default => 2
+
+      # Set max interval in seconds between bulk retries.
+      mod.config :retry_max_interval, :validate => :number, :default => 64
+
+      # The number of times Elasticsearch should internally retry an update/upserted document
+      # See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
+      # for more info
+      mod.config :retry_on_conflict, :validate => :number, :default => 1
+
+      # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
+      # here like `pipeline => "%{INGEST_PIPELINE}"`
+      mod.config :pipeline, :validate => :string, :default => nil
+
+
+      # -----
+      # ILM configurations (beta)
+      # -----
+      # Flag for enabling Index Lifecycle Management integration.
+      mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto'
+
+      # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
+      mod.config :ilm_rollover_alias, :validate => :string, :default => DEFAULT_ROLLOVER_ALIAS
+
+      # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
+      # {now/d} is date math, and will insert the appropriate value automatically.
+      mod.config :ilm_pattern, :validate => :string, :default => '{now/d}-000001'
+
+      # ILM policy to use, if undefined the default policy will be used.
+      mod.config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
+
+    end
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json

@@ -0,0 +1,14 @@
+{
+  "policy" : {
+    "phases": {
+      "hot" : {
+        "actions" : {
+          "rollover" : {
+            "max_size" : "50gb",
+            "max_age":"30d"
+          }
+        }
+      }
+    }
+  }
+}