logstash-output-elasticsearch-test 11.16.0-x86_64-linux

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -0,0 +1,92 @@
+module LogStash; module Outputs; class ElasticSearch
+  module Ilm
+
+    ILM_POLICY_PATH = "default-ilm-policy.json"
+
+    def setup_ilm
+      logger.warn("Overwriting supplied index #{@index} with rollover alias #{@ilm_rollover_alias}") unless default_index?(@index)
+      @index = @ilm_rollover_alias
+      maybe_create_rollover_alias
+      maybe_create_ilm_policy
+    end
+
+    def ilm_in_use?
+      return @ilm_actually_enabled if defined?(@ilm_actually_enabled)
+      @ilm_actually_enabled =
+        begin
+          if serverless?
+            raise LogStash::ConfigurationError, "Invalid ILM configuration `ilm_enabled => true`. " +
+              "Serverless Elasticsearch cluster does not support Index Lifecycle Management." if @ilm_enabled.to_s == 'true'
+            @logger.info("ILM auto configuration (`ilm_enabled => auto` or unset) resolved to `false`. "\
+              "Serverless Elasticsearch cluster does not support Index Lifecycle Management.") if @ilm_enabled == 'auto'
+            false
+          elsif @ilm_enabled == 'auto'
+            if ilm_on_by_default?
+              ilm_alias_set?
+            else
+              @logger.info("ILM auto configuration (`ilm_enabled => auto` or unset) resolved to `false`."\
+                " Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
+              false
+            end
+          elsif @ilm_enabled.to_s == 'true'
+            ilm_alias_set?
+          else
+            false
+          end
+        end
+    end
+
+    private
+
+    def ilm_alias_set?
+      default_index?(@index) || !default_rollover_alias?(@ilm_rollover_alias)
+    end
+
+    def ilm_on_by_default?
+      maximum_seen_major_version >= 7
+    end
+
+    def default_index?(index)
+      index == @default_index
+    end
+
+    def default_rollover_alias?(rollover_alias)
+      rollover_alias == default_ilm_rollover_alias
+    end
+
+    def ilm_policy_default?
+      ilm_policy == LogStash::Outputs::ElasticSearch::DEFAULT_POLICY
+    end
+
+    def maybe_create_ilm_policy
+      if ilm_policy_default?
+        client.ilm_policy_put(ilm_policy, policy_payload) unless client.ilm_policy_exists?(ilm_policy)
+      else
+        raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance" unless client.ilm_policy_exists?(ilm_policy)
+      end
+    end
+
+    def maybe_create_rollover_alias
+      client.rollover_alias_put(rollover_alias_target, rollover_alias_payload) unless client.rollover_alias_exists?(ilm_rollover_alias)
+    end
+
+    def rollover_alias_target
+      "<#{ilm_rollover_alias}-#{ilm_pattern}>"
+    end
+
+    def rollover_alias_payload
+      {
+          'aliases' => {
+              ilm_rollover_alias =>{
+                  'is_write_index' =>  true
+              }
+          }
+      }
+    end
+
+    def policy_payload
+      policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__))
+      LogStash::Json.load(::IO.read(policy_path))
+    end
+  end
+end; end; end

data/lib/logstash/outputs/elasticsearch/license_checker.rb

@@ -0,0 +1,52 @@
+module LogStash; module Outputs; class ElasticSearch
+  class LicenseChecker
+
+    def initialize(logger)
+      @logger = logger
+    end
+
+    # Figure out if the provided license is appropriate or not
+    # The appropriate_license? methods is the method called from LogStash::Outputs::ElasticSearch::HttpClient::Pool#healthcheck!
+    # @param pool
+    # @param url [LogStash::Util::SafeURI] ES node URL
+    # @return [Boolean] true if provided license is deemed appropriate
+    def appropriate_license?(pool, url)
+      return true if pool.serverless?
+
+      license = extract_license(pool.get_license(url))
+      case license_status(license)
+      when 'active'
+        true
+      when nil
+        warn_no_license(url)
+        false
+      else # 'invalid', 'expired'
+        warn_invalid_license(url, license)
+        true
+      end
+    end
+
+    NO_LICENSE = {}.freeze
+    private_constant :NO_LICENSE
+
+    def extract_license(license)
+      license.fetch("license", NO_LICENSE)
+    end
+
+    def license_status(license)
+      license.fetch("status", nil)
+    end
+
+    private
+
+    def warn_no_license(url)
+      @logger.error("Could not connect to a compatible version of Elasticsearch", url: url.sanitized.to_s)
+    end
+
+    def warn_invalid_license(url, license)
+      @logger.warn("Elasticsearch license is not active, please check Elasticsearch’s licensing information",
+                   url: url.sanitized.to_s, license: license)
+    end
+
+  end
+end; end; end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -0,0 +1,131 @@
+module LogStash; module Outputs; class ElasticSearch
+  class TemplateManager
+    LEGACY_TEMPLATE_ENDPOINT = '_template'.freeze
+    INDEX_TEMPLATE_ENDPOINT = '_index_template'.freeze
+
+    # To be mixed into the elasticsearch plugin base
+    def self.install_template(plugin)
+      return unless plugin.manage_template
+
+      if plugin.maximum_seen_major_version < 8 && plugin.template_api == 'auto'
+        plugin.logger.warn("`template_api => auto` resolved to `legacy` since we are connected to " + "Elasticsearch #{plugin.maximum_seen_major_version}, " +
+                           "but will resolve to `composable` the first time it connects to Elasticsearch 8+. " +
+                           "We recommend either setting `template_api => legacy` to continue providing legacy-style templates, " +
+                           "or migrating your template to the composable style and setting `template_api => composable`. " +
+                           "The legacy template API is slated for removal in Elasticsearch 9.")
+      elsif plugin.template_api == 'legacy' && plugin.serverless?
+        raise LogStash::ConfigurationError, "Invalid template configuration `template_api => legacy`. Serverless Elasticsearch does not support legacy template API."
+      end
+
+
+      if plugin.template
+        plugin.logger.info("Using mapping template from", :path => plugin.template)
+        template = read_template_file(plugin.template)
+      else
+        plugin.logger.info("Using a default mapping template", :es_version => plugin.maximum_seen_major_version,
+                                                               :ecs_compatibility => plugin.ecs_compatibility)
+        template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility)
+      end
+
+      add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
+      plugin.logger.debug("Attempting to install template", template: template)
+      install(plugin.client, template_endpoint(plugin), template_name(plugin), template, plugin.template_overwrite)
+    end
+
+    private
+    def self.load_default_template(es_major_version, ecs_compatibility)
+      template_path = default_template_path(es_major_version, ecs_compatibility)
+      read_template_file(template_path)
+    rescue => e
+      fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
+    end
+
+    def self.install(client, template_endpoint, template_name, template, template_overwrite)
+      client.template_install(template_endpoint, template_name, template, template_overwrite)
+    end
+
+    def self.add_ilm_settings_to_template(plugin, template)
+      # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern
+      # definition - remove any existing definition of 'template'
+      template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8
+      template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
+      settings = resolve_template_settings(plugin, template)
+      if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias'])
+        plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled")
+      end
+      settings.update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
+    end
+
+    def self.resolve_template_settings(plugin, template)
+      if template.key?('template')
+        plugin.logger.trace("Resolving ILM template settings: under 'template' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version)
+        composable_index_template_settings(template)
+      elsif template.key?('settings')
+        plugin.logger.trace("Resolving ILM template settings: under 'settings' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version)
+        legacy_index_template_settings(template)
+      else
+        use_index_template_api = index_template_api?(plugin)
+        plugin.logger.trace("Resolving ILM template settings: template doesn't have 'settings' or 'template' fields, falling back to auto detection", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version, :index_template_api => use_index_template_api)
+        if use_index_template_api
+          composable_index_template_settings(template)
+        else
+          legacy_index_template_settings(template)
+        end
+      end
+    end
+
+    # Sets ['settings'] field to be compatible with _template API structure
+    def self.legacy_index_template_settings(template)
+      template['settings'] ||= {}
+    end
+
+    # Sets the ['template']['settings'] fields if not exist to be compatible with _index_template API structure
+    def self.composable_index_template_settings(template)
+      template['template'] ||= {}
+      template['template']['settings'] ||= {}
+    end
+
+    # Template name - if template_name set, use it
+    #                 if not and ILM is enabled, use the rollover alias
+    #                 else use the default value of template_name
+    def self.template_name(plugin)
+      plugin.ilm_in_use? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name
+    end
+
+    def self.default_template_path(es_major_version, ecs_compatibility=:disabled)
+      template_version = es_major_version
+      default_template_name = "templates/ecs-#{ecs_compatibility}/elasticsearch-#{template_version}x.json"
+      ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
+    end
+
+    def self.read_template_file(template_path)
+      raise ArgumentError, "Template file '#{template_path}' could not be found" unless ::File.exists?(template_path)
+      template_data = ::IO.read(template_path)
+      LogStash::Json.load(template_data)
+    end
+
+    def self.template_endpoint(plugin)
+      index_template_api?(plugin) ? INDEX_TEMPLATE_ENDPOINT : LEGACY_TEMPLATE_ENDPOINT
+    end
+
+    def self.index_template_api?(plugin)
+      case plugin.serverless?
+      when true
+        true
+      else
+        case plugin.template_api
+        when 'auto'
+          plugin.maximum_seen_major_version >= 8
+        when 'composable'
+          true
+        when 'legacy'
+          false
+        else
+          plugin.logger.warn("Invalid template_api value #{plugin.template_api}")
+          true
+        end
+      end
+    end
+
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json

@@ -0,0 +1,45 @@
+{
+  "template" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date"},
+        "@version": { "type": "keyword"},
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json

@@ -0,0 +1,44 @@
+{
+  "index_patterns" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s",
+    "number_of_shards": 1
+  },
+  "mappings" : {
+    "dynamic_templates" : [ {
+      "message_field" : {
+        "path_match" : "message",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text",
+          "norms" : false
+        }
+      }
+    }, {
+      "string_fields" : {
+        "match" : "*",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text", "norms" : false,
+          "fields" : {
+            "keyword" : { "type": "keyword", "ignore_above": 256 }
+          }
+        }
+      }
+    } ],
+    "properties" : {
+      "@timestamp": { "type": "date"},
+      "@version": { "type": "keyword"},
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json

@@ -0,0 +1,50 @@
+{
+  "index_patterns" : "logstash-*",
+  "version" : 80001,
+  "template" : {
+    "settings" : {
+      "index.refresh_interval" : "5s",
+      "number_of_shards": 1
+    },
+    "mappings" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date" },
+        "@version": { "type": "keyword" },
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  },
+  "priority": 200,
+  "_meta" : {
+    "description": "index template for logstash-output-elasticsearch"
+  }
+}