logstash-output-opensearch 1.0.0-java

data/lib/logstash/outputs/opensearch/http_client_builder.rb

@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require 'cgi'
+require "base64"
+
+module LogStash; module Outputs; class OpenSearch;
+  module HttpClientBuilder
+    def self.build(logger, hosts, params)
+      client_settings = {
+        :pool_max => params["pool_max"],
+        :pool_max_per_route => params["pool_max_per_route"],
+        :check_connection_timeout => params["validate_after_inactivity"],
+        :http_compression => params["http_compression"],
+        :headers => params["custom_headers"] || {}
+      }
+      
+      client_settings[:proxy] = params["proxy"] if params["proxy"]
+      
+      common_options = {
+        :distribution_checker => params["distribution_checker"],
+        :client_settings => client_settings,
+        :metric => params["metric"],
+        :resurrect_delay => params["resurrect_delay"]
+      }
+
+      if params["sniffing"]
+        common_options[:sniffing] = true
+        common_options[:sniffer_delay] = params["sniffing_delay"]
+      end
+
+      common_options[:timeout] = params["timeout"] if params["timeout"]
+
+      if params["path"]
+        client_settings[:path] = dedup_slashes("/#{params["path"]}/")
+      end
+
+      common_options[:bulk_path] = if params["bulk_path"]
+         dedup_slashes("/#{params["bulk_path"]}")
+      else
+         dedup_slashes("/#{params["path"]}/_bulk")
+      end
+
+      common_options[:sniffing_path] = if params["sniffing_path"]
+         dedup_slashes("/#{params["sniffing_path"]}")
+      else
+         dedup_slashes("/#{params["path"]}/_nodes/http")
+      end
+
+      common_options[:healthcheck_path] = if params["healthcheck_path"]
+         dedup_slashes("/#{params["healthcheck_path"]}")
+      else
+         dedup_slashes("/#{params["path"]}")
+      end
+
+      if params["parameters"]
+        client_settings[:parameters] = params["parameters"]
+      end
+
+      logger.debug? && logger.debug("Normalizing http path", :path => params["path"], :normalized => client_settings[:path])
+
+      client_settings.merge! setup_ssl(logger, params)
+      common_options.merge! setup_basic_auth(logger, params)
+
+      external_version_types = ["external", "external_gt", "external_gte"]
+      # External Version validation
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning requires the presence of a version number."
+      ) if external_version_types.include?(params.fetch('version_type', '')) and params.fetch("version", nil) == nil
+ 
+
+      # Create API setup
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning is not supported by the create action."
+      ) if params['action'] == 'create' and external_version_types.include?(params.fetch('version_type', ''))
+
+      # Update API setup
+      raise( LogStash::ConfigurationError,
+        "doc_as_upsert and scripted_upsert are mutually exclusive."
+      ) if params["doc_as_upsert"] and params["scripted_upsert"]
+
+      raise(
+        LogStash::ConfigurationError,
+        "Specifying action => 'update' needs a document_id."
+      ) if params['action'] == 'update' and params.fetch('document_id', '') == ''
+
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning is not supported by the update action."
+      ) if params['action'] == 'update' and external_version_types.include?(params.fetch('version_type', ''))
+
+      # Update API setup
+      update_options = {
+        :doc_as_upsert => params["doc_as_upsert"],
+        :script_var_name => params["script_var_name"],
+        :script_type => params["script_type"],
+        :script_lang => params["script_lang"],
+        :scripted_upsert => params["scripted_upsert"]
+      }
+      common_options.merge! update_options if params["action"] == 'update'
+
+      create_http_client(common_options.merge(:hosts => hosts, :logger => logger))
+    end
+
+    def self.create_http_client(options)
+      LogStash::Outputs::OpenSearch::HttpClient.new(options)
+    end
+
+    def self.setup_ssl(logger, params)
+      params["ssl"] = true if params["hosts"].any? {|h| h.scheme == "https" }
+      return {} if params["ssl"].nil?
+
+      return {:ssl => {:enabled => false}} if params["ssl"] == false
+
+      cacert, truststore, truststore_password, keystore, keystore_password, tls_client_cert, tls_client_key =
+        params.values_at('cacert', 'truststore', 'truststore_password', 'keystore', 'keystore_password', 'tls_certificate', 'tls_key')
+
+      if cacert && truststore
+        raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if truststore
+      end
+
+      if (tls_client_cert && !tls_client_key)
+        raise(LogStash::ConfigurationError, "\"tls_key\" is missing")
+      end
+
+      if (!tls_client_cert && tls_client_key)
+        raise(LogStash::ConfigurationError, "\"tls_certificate\" is missing")
+      end
+
+      ssl_options = {:enabled => true}
+
+      if cacert
+        ssl_options[:ca_file] = cacert
+      elsif truststore
+        ssl_options[:truststore_password] = truststore_password.value if truststore_password
+      end
+
+      ssl_options[:truststore] = truststore if truststore
+      if keystore
+        ssl_options[:keystore] = keystore
+        ssl_options[:keystore_password] = keystore_password.value if keystore_password
+      end
+      if (tls_client_cert && tls_client_key)
+        ssl_options[:client_cert] = tls_client_cert
+        ssl_options[:client_key] = tls_client_key
+      end
+      if !params["ssl_certificate_verification"]
+        logger.warn [
+                       "** WARNING ** Detected UNSAFE options in opensearch output configuration!",
+                       "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
+                       "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
+                     ].join("\n")
+        ssl_options[:verify] = false
+      end
+      { ssl: ssl_options }
+    end
+
+    def self.setup_basic_auth(logger, params)
+      user, password = params["user"], params["password"]
+      
+      return {} unless user && password && password.value
+
+      {
+        :user => CGI.escape(user),
+        :password => CGI.escape(password.value)
+      }
+    end
+
+    private
+    def self.dedup_slashes(url)
+      url.gsub(/\/+/, "/")
+    end
+  end
+end; end; end

data/lib/logstash/outputs/opensearch/template_manager.rb

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+module LogStash; module Outputs; class OpenSearch
+  class TemplateManager
+    # To be mixed into the opensearch plugin base
+    def self.install_template(plugin)
+      return unless plugin.manage_template
+      if plugin.template
+        plugin.logger.info("Using mapping template from", :path => plugin.template)
+        template = read_template_file(plugin.template)
+      else
+        plugin.logger.info("Using a default mapping template", :version => plugin.maximum_seen_major_version,
+                                                               :ecs_compatibility => plugin.ecs_compatibility)
+        template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility)
+      end
+
+      plugin.logger.debug("Attempting to install template", template: template)
+      install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
+    end
+
+    private
+    def self.load_default_template(major_version, ecs_compatibility)
+      template_path = default_template_path(major_version, ecs_compatibility)
+      read_template_file(template_path)
+    rescue => e
+      fail "Failed to load default template for OpenSearch v#{major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
+    end
+
+    def self.install(client, template_name, template, template_overwrite)
+      client.template_install(template_name, template, template_overwrite)
+    end
+
+    def self.template_settings(plugin, template)
+      template['settings']
+    end
+
+    def self.template_name(plugin)
+      plugin.template_name
+    end
+
+    def self.default_template_path(major_version, ecs_compatibility=:disabled)
+      template_version = major_version
+      default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x.json"
+      ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
+    end
+
+    def self.read_template_file(template_path)
+      raise ArgumentError, "Template file '#{template_path}' could not be found" unless ::File.exists?(template_path)
+      template_data = ::IO.read(template_path)
+      LogStash::Json.load(template_data)
+    end
+  end
+end end end

data/lib/logstash/outputs/opensearch/templates/ecs-disabled/1x.json

@@ -0,0 +1,44 @@
+{
+  "index_patterns" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s",
+    "number_of_shards": 1
+  },
+  "mappings" : {
+    "dynamic_templates" : [ {
+      "message_field" : {
+        "path_match" : "message",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text",
+          "norms" : false
+        }
+      }
+    }, {
+      "string_fields" : {
+        "match" : "*",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text", "norms" : false,
+          "fields" : {
+            "keyword" : { "type": "keyword", "ignore_above": 256 }
+          }
+        }
+      }
+    } ],
+    "properties" : {
+      "@timestamp": { "type": "date"},
+      "@version": { "type": "keyword"},
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/opensearch/templates/ecs-disabled/7x.json

@@ -0,0 +1,44 @@
+{
+  "index_patterns" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s",
+    "number_of_shards": 1
+  },
+  "mappings" : {
+    "dynamic_templates" : [ {
+      "message_field" : {
+        "path_match" : "message",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text",
+          "norms" : false
+        }
+      }
+    }, {
+      "string_fields" : {
+        "match" : "*",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text", "norms" : false,
+          "fields" : {
+            "keyword" : { "type": "keyword", "ignore_above": 256 }
+          }
+        }
+      }
+    } ],
+    "properties" : {
+      "@timestamp": { "type": "date"},
+      "@version": { "type": "keyword"},
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/plugin_mixins/opensearch/api_configs.rb

@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+module LogStash; module PluginMixins; module OpenSearch
+  module APIConfigs
+
+    # This module defines common options that can be reused by alternate OpenSearch output plugins.
+
+    DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//127.0.0.1")
+
+    CONFIG_PARAMS = {
+        # Username to authenticate to a secure OpenSearch cluster
+        :user => { :validate => :string },
+        # Password to authenticate to a secure OpenSearch cluster
+        :password => { :validate => :password },
+
+        # The document ID for the index. Useful for overwriting existing entries in
+        # OpenSearch with the same ID.
+        :document_id => { :validate => :string },
+
+        # HTTP Path at which the OpenSearch server lives. Use this if you must run OpenSearch behind a proxy that remaps
+        # the root path for the OpenSearch HTTP API lives.
+        # Note that if you use paths as components of URLs in the 'hosts' field you may
+        # not also set this field. That will raise an error at startup
+        :path => { :validate => :string },
+
+        # HTTP Path to perform the _bulk requests to
+        # this defaults to a concatenation of the path parameter and "_bulk"
+        :bulk_path => { :validate => :string },
+
+        # Pass a set of key value pairs as the URL query string. This query string is added
+        # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
+        # urls that already have query strings, the one specified here will be appended.
+        :parameters => { :validate => :hash },
+
+        # Enable SSL/TLS secured communication to OpenSearch cluster. Leaving this unspecified will use whatever scheme
+        # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
+        # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
+        :ssl => { :validate => :boolean },
+
+        # Option to validate the server's certificate. Disabling this severely compromises security.
+        # For more information on disabling certificate verification please read
+        # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
+        :ssl_certificate_verification => { :validate => :boolean, :default => true },
+
+        # The .cer or .pem file to validate the server's certificate
+        :cacert => { :validate => :path },
+
+        # The JKS truststore to validate the server's certificate.
+        # Use either `:truststore` or `:cacert`
+        :truststore => { :validate => :path },
+
+        # Set the truststore password
+        :truststore_password => { :validate => :password },
+
+        # The keystore used to present a certificate to the server.
+        # It can be either .jks or .p12
+        :keystore => { :validate => :path },
+
+        # Set the keystore password
+        :keystore_password => { :validate => :password },
+
+        # Set the TLS Client certificate file
+        :tls_certificate => { :validate => :path },
+
+        # Private key file name
+        :tls_key => { :validate => :path },
+
+        # This setting asks OpenSearch for the list of all cluster nodes and adds them to the hosts list.
+        # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
+        # this with master nodes, you probably want to disable HTTP on them by setting
+        # `http.enabled` to false in their OpenSearch.yml. You can either use the `sniffing` option or
+        # manually enter multiple OpenSearch hosts using the `hosts` parameter.
+        :sniffing => { :validate => :boolean, :default => false },
+
+        # How long to wait, in seconds, between sniffing attempts
+        :sniffing_delay => { :validate => :number, :default => 5 },
+
+        # HTTP Path to be used for the sniffing requests
+        # the default value is computed by concatenating the path value and "_nodes/http"
+        # if sniffing_path is set it will be used as an absolute path
+        # do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
+        :sniffing_path => { :validate => :string },
+
+        # Set the address of a forward HTTP proxy.
+        # This used to accept hashes as arguments but now only accepts
+        # arguments of the URI type to prevent leaking credentials.
+        :proxy => { :validate => :uri }, # but empty string is allowed
+
+        # Set the timeout, in seconds, for network operations and requests sent OpenSearch. If
+        # a timeout occurs, the request will be retried.
+        :timeout => { :validate => :number, :default => 60 },
+
+        # Set the OpenSearch errors in the whitelist that you don't want to log.
+        # A useful example is when you want to skip all 409 errors
+        # which are `document_already_exists_exception`.
+        :failure_type_logging_whitelist => { :validate => :array, :default => [] },
+
+        # While the output tries to reuse connections efficiently we have a maximum.
+        # This sets the maximum number of open connections the output will create.
+        # Setting this too low may mean frequently closing / opening connections
+        # which is bad.
+        :pool_max => { :validate => :number, :default => 1000 },
+
+        # While the output tries to reuse connections efficiently we have a maximum per endpoint.
+        # This sets the maximum number of open connections per endpoint the output will create.
+        # Setting this too low may mean frequently closing / opening connections
+        # which is bad.
+        :pool_max_per_route => { :validate => :number, :default => 100 },
+
+        # HTTP Path where a HEAD request is sent when a backend is marked down
+        # the request is sent in the background to see if it has come back again
+        # before it is once again eligible to service requests.
+        # If you have custom firewall rules you may need to change this
+        :healthcheck_path => { :validate => :string },
+
+        # How frequently, in seconds, to wait between resurrection attempts.
+        # Resurrection is the process by which backend endpoints marked 'down' are checked
+        # to see if they have come back to life
+        :resurrect_delay => { :validate => :number, :default => 5 },
+
+        # How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
+        # You may want to set this lower, if you get connection errors regularly
+        # Quoting the Apache commons docs (this client is based Apache Commmons):
+        # 'Defines period of inactivity in milliseconds after which persistent connections must
+        # be re-validated prior to being leased to the consumer. Non-positive value passed to
+        # this method disables connection validation. This check helps detect connections that
+        # have become stale (half-closed) while kept inactive in the pool.'
+        # See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
+        :validate_after_inactivity => { :validate => :number, :default => 10000 },
+
+        # Enable gzip compression on requests.
+        :http_compression => { :validate => :boolean, :default => false },
+
+        # Custom Headers to send on each request to OpenSearch nodes
+        :custom_headers => { :validate => :hash, :default => {} },
+
+        # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
+        # Remember the `http` protocol uses the http address (eg. 9200, not 9300).
+        #     `"127.0.0.1"`
+        #     `["127.0.0.1:9200","127.0.0.2:9200"]`
+        #     `["http://127.0.0.1"]`
+        #     `["https://127.0.0.1:9200"]`
+        #     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
+        # It is important to exclude dedicated master nodes from the `hosts` list
+        # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in OpenSearch.
+        #
+        # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
+        :hosts => { :validate => :uri, :default => [ DEFAULT_HOST ], :list => true },
+
+        # Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
+        :retry_initial_interval => { :validate => :number, :default => 2 },
+
+        # Set max interval in seconds between bulk retries.
+        :retry_max_interval => { :validate => :number, :default => 64 }
+    }.freeze
+
+    def self.included(base)
+      CONFIG_PARAMS.each { |name, opts| base.config(name, opts) }
+    end
+  end
+end; end; end