logstash-filter-elastic_integration 0.0.2-java → 0.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e763ac8238ed2ebca641f89dde3596d76b63f2e93f9214894c93add947c0fb14
-  data.tar.gz: acb232f4b3481f47684421ccfffdda198a4141ec5209b67f881f04ca28edd539
+  metadata.gz: 844b5987e2e5e28d7e560c80eeec166c3d07651a6b4d167dc449e9639beda926
+  data.tar.gz: 6f85d9b9cabda183c6a365f3030876c21990161dbe64e1e82e8d349dccea4b17
 SHA512:
-  metadata.gz: d563ccc980501c370b2b95eecbfbd8f43ca9cc30d792d3ab8122cc08bfd227c1a4bfa41dcea7041f502afb25d4a3f391981309157b5a5bddf8b92a37543167f3
-  data.tar.gz: 3e75a0255e24a4c67ef04f65e7bdeb137c233d9c731624b0436c26a3f1ffe0bf6274b52090a924b072f602678b6af7570c105f590b81e13edf61bfdff1df8936
+  metadata.gz: a0e711a5d52416b93fbf88b0658657722b5a63d3be91f1a76125123ff614826705aa9a065f014a2a00489729c15b8ec1a9d654d3ab74235f53e2e4d66aec2225
+  data.tar.gz: 0af45a4b9be67629091d3a45d6dd7f8c666cc403ff0e1dee8aa1244a5b27cfc843340b0a61a817181f14b14982786a0b298014936b05fbb3c53aa611e0165fe2

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.1.0

data/lib/logstash/filters/elastic_integration/geoip_database_provider_bridge.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+
+########################################################################
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V.
+# under one or more contributor license agreements. Licensed under the
+# Elastic License 2.0; you may not use this file except in compliance
+# with the Elastic License 2.0.
+########################################################################
+
+require_relative "jar_dependencies"
+
+##
+# This module encapsulates methods and classes for bridging the gap between the
+# Ruby-API [LogStash::GeoipDatabaseManager] and this plugin's native-Java
+# GeoipDatabaseProvider.
+module LogStash::Filters::ElasticIntegration::GeoipDatabaseProviderBridge
+
+  GUIDANCE = "integrations that rely on the Geoip Processor will be unable to enrich events with geo data "\
+             "unless you either provide your own databases with `geoip_database_directory` or run this pipeline "\
+             "in a Logstash with Geoip Database Management enabled."
+
+  def initialize_geoip_database_provider!
+    java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpDatabaseProvider')
+    @geoip_database_provider ||= GeoIpDatabaseProvider::Builder.new.tap do |builder|
+      if geoip_database_directory
+        logger.debug("discovering geoip databases from #{geoip_database_directory}")
+        builder.discoverDatabases(java.io.File.new(geoip_database_directory))
+      else
+        geoip_database_manager = load_geoip_database_manager!
+        if :UNAVAILABLE == geoip_database_manager
+          logger.warn("Geoip Database Management is not available in the running version of Logstash; #{GUIDANCE}")
+        elsif geoip_database_manager.enabled?
+          logger.info "by not manually configuring self-managed databases with `geoip_database_directory => ...` "\
+                        "you accept and agree to the MaxMind EULA, which allows Elastic Integrations to use Logstash's Geoip Database Management service. "\
+                        "For more details please visit https://www.maxmind.com/en/geolite2/eula"
+
+          geoip_database_manager.supported_database_types.each do |type|
+            logger.debug("subscribing to managed geoip database #{type}")
+            builder.setDatabaseHolder("GeoLite2-#{type}.mmdb", ObservingDatabaseHolder.new(type, eula_manager: geoip_database_manager, logger: logger))
+          end
+        elsif geoip_database_directory.nil?
+          logger.warn("Geoip Database Management is disabled; #{GUIDANCE}")
+        end
+      end
+    end.build
+  end
+
+  def load_geoip_database_manager!
+    require 'geoip_database_management/manager'
+
+    LogStash::GeoipDatabaseManagement::Manager.instance
+  rescue LoadError
+    :UNAVAILABLE
+  end
+
+  java_import('co.elastic.logstash.filters.elasticintegration.geoip.ManagedGeoipDatabaseHolder')
+  class ObservingDatabaseHolder < ManagedGeoipDatabaseHolder
+    def initialize(simple_database_type, eula_manager:, logger: nil)
+      super("GeoLite2-#{simple_database_type}")
+
+      @simple_database_type = simple_database_type
+      @logger = logger
+
+      @subscription = eula_manager.subscribe_database_path(simple_database_type)
+      @subscription.observe(self)
+    end
+
+    def construct(db_info)
+      @logger&.debug("CONSTRUCT[#{@simple_database_type} => #{db_info}]")
+      self.setDatabasePath(db_info.path)
+    end
+
+    def on_update(db_info)
+      @logger&.debug("ON_UPDATE[#{@simple_database_type} => #{db_info}]")
+      self.setDatabasePath(db_info.path)
+    end
+
+    def on_expire()
+      @logger&.debug("ON_EXPIRE[#{@simple_database_type}]")
+      self.setDatabasePath(nil)
+    end
+
+    def close
+      super
+    ensure
+      @subscription&.release!
+    end
+
+  end
+end

data/lib/logstash/filters/elastic_integration/jar_dependencies.rb

@@ -8,4 +8,4 @@
 ########################################################################
 
 require 'jar_dependencies'
-require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.2')
+require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.1.0')

data/lib/logstash/filters/elastic_integration.rb

@@ -18,6 +18,7 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
   ELASTICSEARCH_DEFAULT_PATH = '/'.freeze
   HTTP_PROTOCOL = "http".freeze
   HTTPS_PROTOCOL = "https".freeze
+  ELASTIC_API_VERSION = "2023-10-31".freeze
 
   # Sets the host(s) of the remote instance. If given an array it will load balance
   # requests across the hosts specified in the `hosts` parameter. Hosts can be any of
@@ -105,8 +106,10 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
 
     require_relative "elastic_integration/jar_dependencies"
     require_relative "elastic_integration/event_api_bridge"
+    require_relative "elastic_integration/geoip_database_provider_bridge"
 
     extend EventApiBridge
+    extend GeoipDatabaseProviderBridge
 
     super
 
@@ -335,10 +338,21 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
 
   def initialize_elasticsearch_rest_client!
     java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
+    java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
 
-    @elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(extract_immutable_config)
+    config = extract_immutable_config
+    @elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(config)
                                                                .map(&:build)
                                                                .orElseThrow() # todo: ruby/java bridge better exception
+
+    if serverless?
+      @elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(config)
+                                                                 .map do |builder|
+                                                                    builder.configureElasticApi { |elasticApi| elasticApi.setApiVersion(ELASTIC_API_VERSION) }
+                                                                  end
+                                                                 .map(&:build)
+                                                                 .orElseThrow()
+    end
   end
 
   def initialize_event_processor!
@@ -353,14 +367,6 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
     raise_config_error!("configuration did not produce an EventProcessor: #{exception}")
   end
 
-  def initialize_geoip_database_provider!
-    java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpDatabaseProvider')
-
-    @geoip_database_provider ||= GeoIpDatabaseProvider::Builder.new.tap do |builder|
-      builder.setDatabases(java.io.File.new(@geoip_database_directory)) if @geoip_database_directory
-    end.build
-  end
-
   def perform_preflight_check!
     java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
 
@@ -397,6 +403,12 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
     raise_config_error!(e.message)
   end
 
+  def serverless?
+    PreflightCheck.new(@elasticsearch_rest_client).isServerless
+  rescue => e
+    raise_config_error!(e.message)
+  end
+
   ##
   # single-use helper to ensure the running Logstash is a _complete_ distro that has
   # non-OSS features active. Runtime detection mechanism relies on LogStash::OSS,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elastic_integration
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-10 00:00:00.000000000 Z
+date: 2023-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,9 +71,10 @@ files:
 - VERSION
 - lib/logstash/filters/elastic_integration.rb
 - lib/logstash/filters/elastic_integration/event_api_bridge.rb
+- lib/logstash/filters/elastic_integration/geoip_database_provider_bridge.rb
 - lib/logstash/filters/elastic_integration/jar_dependencies.rb
 - logstash-filter-elastic_integration.gemspec
-- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.2/logstash-filter-elastic_integration-0.0.2.jar
+- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.1.0/logstash-filter-elastic_integration-0.1.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - ELv2
@@ -100,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Processes Elastic Integrations