logstash-filter-elastic_integration 0.0.1-java → 0.0.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c797a3d5774cab43738ee098605fcba0a8df86d104b5334d38b27f882e7b086
-  data.tar.gz: 43d2a785e22bf742e3e3fb9ba0ae93a7b2ede62ed6a25fae590fbd6c869eb729
+  metadata.gz: 245850625d596870d7536a1282f78419735370cf2d4b4a4d5ad51d59872f7de9
+  data.tar.gz: '017508efe2333faa598921431926a80f58cfaf7f2dae9c6b5a2d53a787bc296b'
 SHA512:
-  metadata.gz: 2ed848a542bf89c151122bdf98d7ca7c85573ee3d81e5019632f2a1be2c61941d704e21342f28c1e52680956a862a823939e43c61d682e59a877bbedd198c0e6
-  data.tar.gz: 3d2231665a2277b8bed047b1fbf412e56d3e84926893769e69277d1e3d400781d41c2e0551b8d21e3b0ba9eaab1621c7514f497c87ea07f402b740d90b1e2fc4
+  metadata.gz: 3234009681fad3833eacd39a943661a61ae4541c4d4fe3df6f5c0f0516ca53bdf6da70eaf5890de460def6c231173c785c1a9a317c82be0275dc8ee79a99cf81
+  data.tar.gz: f5bc0d1effe8c21c5c7cdc69ffe265d8cbcc10f67ee32e4e64d43ea4d494618db78edf8d5e42d05e644bfb87d2862982fddc8df571f0d366a6bcb6685b16e545

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.3

data/lib/logstash/filters/elastic_integration/jar_dependencies.rb

@@ -8,4 +8,4 @@
 ########################################################################
 
 require 'jar_dependencies'
-require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.1')
+require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.3')

data/lib/logstash/filters/elastic_integration.rb

@@ -10,25 +10,15 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 
-require_relative "elastic_integration/jar_dependencies"
-
 class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
 
-  require_relative "elastic_integration/event_api_bridge"
-  include EventApiBridge
-
   config_name "elastic_integration"
 
-  java_import('co.elastic.logstash.filters.elasticintegration.PluginConfiguration')
-  java_import('co.elastic.logstash.filters.elasticintegration.EventProcessor')
-  java_import('co.elastic.logstash.filters.elasticintegration.EventProcessorBuilder')
-  java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
-  java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
-
   ELASTICSEARCH_DEFAULT_PORT = 9200.freeze
   ELASTICSEARCH_DEFAULT_PATH = '/'.freeze
   HTTP_PROTOCOL = "http".freeze
   HTTPS_PROTOCOL = "https".freeze
+  ELASTIC_API_VERSION = "2023-10-31".freeze
 
   # Sets the host(s) of the remote instance. If given an array it will load balance
   # requests across the hosts specified in the `hosts` parameter. Hosts can be any of
@@ -86,10 +76,10 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
   config :ssl_keystore_password, :validate => :password
 
   # Username for basic authentication
-  config :auth_basic_username, :validate => :string
+  config :username, :validate => :string
 
   # Password for basic authentication
-  config :auth_basic_password, :validate => :password
+  config :password, :validate => :password
 
   # Cloud authentication string ("<username>:<password>" format) to connect to Elastic cloud.
   #
@@ -99,23 +89,30 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
   # A key to authenticate when connecting to Elasticsearch
   config :api_key, :validate => :password
 
-  # A directory containing one or more Maxmind Datbase files in *.mmdb format
+  # A directory containing one or more Maxmind Database files in *.mmdb format
   config :geoip_database_directory, :validate => :path
 
+  # a sprintf template for resolving the pipeline name; when this template does
+  # not fully-resolve no pipeline will be run.
+  config :pipeline_name, :validate => :string
 
+  ##
+  # Validates that this plugin can be initialized BEFORE loading dependencies
+  # and delegating to super, so that when this plugin CANNOT be run the process
+  # is not encumbered by those dependencies.
   def initialize(*a, &b)
-    # This Elastic-licensed plugin needs to run in a _complete_ distro of Logstash that
-    # has non-OSS features active. Runtime detection mechanism relies on LogStash::OSS,
-    # which is set in the prelude to LogStash::Runner, and is bypassed when LogStash::OSS
-    # is not defined (such as when running specs from source)
-    if defined?(LogStash::OSS) && LogStash::OSS
-      raise_config_error! <<~ERR
-        The Elastic Integration filter for Logstash is an Elastic-licensed plugin
-        that REQUIRES the complete Logstash distribution, including non-OSS features.
-      ERR
-    end
+    ensure_complete_logstash!
+    ensure_java_major_version!(17)
+
+    require_relative "elastic_integration/jar_dependencies"
+    require_relative "elastic_integration/event_api_bridge"
+
+    extend EventApiBridge
 
     super
+
+    java_import('co.elastic.logstash.filters.elasticintegration.util.PluginContext')
+    @plugin_context = PluginContext.new(execution_context&.pipeline_id || "UNDEF", id)
   end
 
   def register
@@ -207,16 +204,16 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
   def validate_auth_settings!
     @cloud_auth           = @cloud_auth&.freeze
     @api_key              = @api_key&.freeze
-    @auth_basic_username  = @auth_basic_username&.freeze
-    @auth_basic_password  = @auth_basic_password&.freeze
+    @username             = @username&.freeze
+    @password             = @password&.freeze
 
-    raise_config_error! "`auth_basic_username` requires `auth_basic_password`" if @auth_basic_username && !@auth_basic_password
-    raise_config_error! "`auth_basic_password` is not allowed unless `auth_basic_username` is specified" if !@auth_basic_username && @auth_basic_password
-    if @auth_basic_username && @auth_basic_password
-      raise_config_error! "Empty `auth_basic_username` or `auth_basic_password` is not allowed" if @auth_basic_username.empty? || @auth_basic_password.value.empty?
+    raise_config_error! "`username` requires `password`" if @username && !@password
+    raise_config_error! "`password` is not allowed unless `username` is specified" if !@username && @password
+    if @username && @password
+      raise_config_error! "Empty `username` or `password` is not allowed" if @username.empty? || @password.value.empty?
     end
 
-    possible_auth_options = original_params.keys & %w(auth_basic_password cloud_auth api_key)
+    possible_auth_options = original_params.keys & %w(password cloud_auth api_key)
     raise_config_error!("Multiple authentication #{possible_auth_options} options cannot be used together. Please provide ONLY one.") if possible_auth_options.size > 1
 
     raise_config_error! "Empty `cloud_auth` is not allowed" if @cloud_auth && @cloud_auth.value.empty?
@@ -303,58 +300,73 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
   ##
   # Builds a `PluginConfiguration` from the previously-validated config
   def extract_immutable_config
-    builder = PluginConfiguration::Builder.new
+    java_import('co.elastic.logstash.filters.elasticintegration.PluginConfiguration')
+    @immutable_config ||= PluginConfiguration::Builder.new.tap do |builder|
+      builder.setId @id
 
-    builder.setId @id
+      builder.setHosts @hosts&.map(&:to_s)
+      builder.setCloudId @cloud_id
 
-    builder.setHosts @hosts&.map(&:to_s)
-    builder.setCloudId @cloud_id
+      builder.setSslEnabled @ssl_enabled
 
-    builder.setSslEnabled @ssl_enabled
+      # ssl trust
+      builder.setSslVerificationMode @ssl_verification_mode
+      builder.setSslTruststorePath @ssl_truststore_path
+      builder.setSslTruststorePassword @ssl_truststore_password
+      builder.setSslCertificateAuthorities @ssl_certificate_authorities
 
-    # ssl trust
-    builder.setSslVerificationMode @ssl_verification_mode
-    builder.setSslTruststorePath @ssl_truststore_path
-    builder.setSslTruststorePassword @ssl_truststore_password
-    builder.setSslCertificateAuthorities @ssl_certificate_authorities
+      # ssl identity
+      builder.setSslKeystorePath @keystore
+      builder.setSslKeystorePassword @ssl_keystore_password
+      builder.setSslCertificate @ssl_certificate
+      builder.setSslKey @ssl_key
+      builder.setSslKeyPassphrase @ssl_key_passphrase
 
-    # ssl identity
-    builder.setSslKeystorePath @keystore
-    builder.setSslKeystorePassword @ssl_keystore_password
-    builder.setSslCertificate @ssl_certificate
-    builder.setSslKey @ssl_key
-    builder.setSslKeyPassphrase @ssl_key_passphrase
+      # request auth
+      builder.setAuthBasicUsername @username
+      builder.setAuthBasicPassword @password
+      builder.setCloudAuth @cloud_auth
+      builder.setApiKey @api_key
 
-    # request auth
-    builder.setAuthBasicUsername @auth_basic_username
-    builder.setAuthBasicPassword @auth_basic_password
-    builder.setCloudAuth @cloud_auth
-    builder.setApiKey @api_key
+      # pipeline resolving
+      builder.setPipelineNameTemplate @pipeline_name
 
-    builder.build
+    end.build
   end
 
   def initialize_elasticsearch_rest_client!
-    @elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(extract_immutable_config)
+    java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
+    java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
+
+    config = extract_immutable_config
+    @elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(config)
                                                                .map(&:build)
                                                                .orElseThrow() # todo: ruby/java bridge better exception
+
+    if serverless?
+      @elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(config)
+                                                                 .map do |builder|
+                                                                    builder.configureElasticApi { |elasticApi| elasticApi.setApiVersion(ELASTIC_API_VERSION) }
+                                                                  end
+                                                                 .map(&:build)
+                                                                 .orElseThrow()
+    end
   end
 
   def initialize_event_processor!
     java_import('co.elastic.logstash.filters.elasticintegration.EventProcessorBuilder')
     java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpProcessorFactory')
 
-    @event_processor = EventProcessorBuilder.fromElasticsearch(@elasticsearch_rest_client)
+    @event_processor = EventProcessorBuilder.fromElasticsearch(@elasticsearch_rest_client, extract_immutable_config)
                                             .setFilterMatchListener(method(:filter_matched_java).to_proc)
                                             .addProcessor("geoip") { GeoIpProcessorFactory.new(@geoip_database_provider) }
-                                            .build("logstash.filter.elastic_integration.#{id}.#{__id__}")
+                                            .build(@plugin_context)
   rescue => exception
     raise_config_error!("configuration did not produce an EventProcessor: #{exception}")
   end
 
   def initialize_geoip_database_provider!
     java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpDatabaseProvider')
-    java_import('co.elastic.logstash.filters.elasticintegration.geoip.StaticGeoIpDatabase')
 
     @geoip_database_provider ||= GeoIpDatabaseProvider::Builder.new.tap do |builder|
       builder.setDatabases(java.io.File.new(@geoip_database_directory)) if @geoip_database_directory
@@ -362,8 +374,83 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
   end
 
   def perform_preflight_check!
-    PreflightCheck.new(@elasticsearch_rest_client).check
+    java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
+
+    check_user_privileges!
+    check_es_cluster_license!
+  end
+
+  def check_user_privileges!
+    PreflightCheck.new(@elasticsearch_rest_client).checkUserPrivileges
+  rescue => e
+    security_error_message = "no handler found for uri [/_security/user/_has_privileges]"
+    if e.message.include?(security_error_message)
+      if @password || @cloud_auth || @api_key
+        cred_desc = case
+                    when @password   then "`username` and `password`"
+                    when @cloud_auth then "`cloud_auth`"
+                    when @api_key    then "`api_key`"
+                    end
+
+        recommend_message = "The Elasticsearch cluster does not have security features enabled but request credentials were provided. Either enable security in Elasticsearch (recommended!) or remove the #{cred_desc} request credentials. "
+        raise_config_error! recommend_message.concat(e.message)
+      else
+        # Elasticsearch cluster security disabled, auth also isn't provided, running plugin unsecurily
+        @logger.warn("`elastic_integration` plugin is unable to verify user privileges. It has started with unsafe mode which may cause unexpected failure. Enabling security in Elasticsearch and using user authentication is recommended.")
+      end
+    else
+      raise_config_error!(e.message)
+    end
+  end
+
+  def check_es_cluster_license!
+    PreflightCheck.new(@elasticsearch_rest_client).checkLicense
   rescue => e
     raise_config_error!(e.message)
   end
+
+  def serverless?
+    PreflightCheck.new(@elasticsearch_rest_client).isServerless
+  rescue => e
+    raise_config_error!(e.message)
+  end
+
+  ##
+  # single-use helper to ensure the running Logstash is a _complete_ distro that has
+  # non-OSS features active. Runtime detection mechanism relies on LogStash::OSS,
+  # which is set in the prelude to LogStash::Runner, and is bypassed when LogStash::OSS
+  # is not defined (such as when running specs from source)
+  def ensure_complete_logstash!
+    if defined?(LogStash::OSS) && LogStash::OSS
+      raise_config_error! <<~ERR.gsub(/\s+/, ' ')
+        The Elastic Integration filter for Logstash is an Elastic-licensed plugin
+        that REQUIRES the complete Logstash distribution, including non-OSS features.
+      ERR
+    end
+  end
+
+  ##
+  # single-use helper for ensuring the running JVM meets the minimum
+  # Java version requirement necessary for loading the included jars
+  # @raise [LogStash::EnvironmentError]
+  def ensure_java_major_version!(minimum_major_version)
+    java_version_string = java.lang.System.getProperty("java.specification.version")
+
+    # MAJOR <= 8 ? 1.MAJOR.MINOR : MAJOR.MINOR
+    # https://rubular.com/r/lLW5iUWN9N9N6Z
+    java_major_version_pattern = /(?:(?:(?<=^1\.)[0-8])|^(?:9|[1-9][0-9]+))(?=\.|$)/
+    java_major_version = java_version_string&.slice(java_major_version_pattern)&.to_i
+
+    if (java_major_version.nil?)
+      fail("Failed to retrieve running JVM's major version")
+    elsif (java_major_version < minimum_major_version)
+      fail(LogStash::EnvironmentError, <<~ERR.gsub(/\s+/, ' '))
+        the #{self.class.config_name} #{self.class.plugin_type} plugin requires
+        Java #{minimum_major_version} or later and cannot be instantiated on the
+        current JVM version `#{java_version_string}`.
+        You can either remove the plugin from your pipeline definition or run
+        Logstash with a supported JVM.
+      ERR
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elastic_integration
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-14 00:00:00.000000000 Z
+date: 2023-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -73,7 +73,7 @@ files:
 - lib/logstash/filters/elastic_integration/event_api_bridge.rb
 - lib/logstash/filters/elastic_integration/jar_dependencies.rb
 - logstash-filter-elastic_integration.gemspec
-- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.1/logstash-filter-elastic_integration-0.0.1.jar
+- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.3/logstash-filter-elastic_integration-0.0.3.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - ELv2