logstash-integration-elastic_enterprise_search 2.2.1 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f0c1374a08bc723fd7452e115e7372c8e0bb416c6e7e2e6d10769b15b571b7c
-  data.tar.gz: 53debfd033dae11b5d5c80809c51c8a6db5997c80f5eeb3901414932c0d48e17
+  metadata.gz: b4473449751c6ce422faa5cdbbcec12e78677a54d40209e397c987d2e2e2d05f
+  data.tar.gz: d1774ea9414e64b243017fa4faac4a53c830b66098208762bb601d9e3a9ac834
 SHA512:
-  metadata.gz: dcb96b16bafc1b85a0e4e375925b8d6ad5e6f3366044e1ea13b4318ab0c5b75b84d342fe76077ddcc30f4b0423f0e47028b474d2012e34570357a07ad9ff94e8
-  data.tar.gz: fab4c000386cca49902de6f11aa392d95032b4fd7897826fc7515dac40012315c2356d1c624485b4689faf229b7455176845ac40cdf3c298e596ad517a545a93
+  metadata.gz: 8ed463da5b8975c4765a4688e684e2596a42161128a6bee3f087eff49a6404180051967e6c96f7893581bd7322ff13bdb2d8251f967b8b35496e4b85503e1f83
+  data.tar.gz: e0fcd1f80a3cd350508d9d78922a67e26fc31e01d0433f2fa764033fc2778925b0a8ebebd48e95802718047162630e2d65124395cbd7c7989a7182ce9482e429

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 3.0.1
+ - Add deprecation log for App Search and Workplace Search. Both products are removed from Elastic Stack in version 9 [#22](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/22)
+
+## 3.0.0
+ - Bumped Enterprise Search clients to version `>= 7.16`, `< 9` [#18](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/18)
+ - Added support to SSL configurations (`ssl_certificate_authorities`, `ssl_truststore_path`, `ssl_truststore_password`, `ssl_truststore_type`, `ssl_verification_mode`, `ssl_supported_protocols` and `ssl_cipher_suites`)
+ - [BREAKING] Swiftype endpoints are no longer supported for both plugins App Search and Workplace Search
+   - The App Search deprecated options `host` and `path` were removed
+ - Fixed the sprintf format support for the Workplace Search `source` configuration
+
 ## 2.2.1
  - Fix, change implementation of connectivity check method to be compatible with version `v8.0+` of Workplace Search [#16](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/16) 
 

data/README.md

@@ -88,6 +88,16 @@ bin/plugin install --no-verify
 ```
 - Start Logstash and proceed to test the plugin
 
+### Thoubleshooting integration test failures
+Integration tests uses some certificates fixtures. These security artifacts has 1 year expiration, so time to time
+they would trigger errors due to bad certificate.
+To regenerate use: 
+```sh
+> cd spec/fixture/certificates
+> ./generate.sh
+```
+Re-run locally the integration tests (with Docker scripts) and if it's green again create a PR to update.
+
 ## Contributing
 
 All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.

data/lib/logstash/outputs/elastic_app_search.rb

@@ -1,13 +1,15 @@
 # encoding: utf-8
-require "logstash/outputs/base"
-require "elastic-app-search"
-require "elastic-enterprise-search"
+require 'logstash/outputs/base'
 require 'logstash/plugin_mixins/deprecation_logger_support'
+require 'logstash/plugin_mixins/enterprise_search/ssl_configs'
+require 'logstash/plugin_mixins/enterprise_search/client'
 
 class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
+
+  include LogStash::PluginMixins::EnterpriseSearch::SSLConfigs
   include LogStash::PluginMixins::DeprecationLoggerSupport
 
-  config_name "elastic_app_search"
+  config_name 'elastic_app_search'
 
   # The name of the search engine you created in App Search, an information
   # repository that includes the indexed document records.
@@ -21,16 +23,11 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
   # or, if the field is missing from the event and cannot be resolved at all.
   config :engine, :validate => :string, :required => true
 
-  # The hostname of the App Search API that is associated with your App Search account.
-  # Set this when using the https://www.elastic.co/cloud/app-search-service
-  config :host, :validate => :string
-
-  # The value of the API endpoint in the form of a URL. Note: The value of the of the `path` setting will be will be appended to this URL.
-  # Set this when using the https://www.elastic.co/downloads/app-search
-  config :url, :validate => :string
+  # The value of the API endpoint in the form of a URL.
+  config :url, :validate => :string, :required => true, :default => Elastic::EnterpriseSearch::Utils::DEFAULT_HOST
 
-  # The private API Key with write permissions. Visit the https://app.swiftype.com/as/credentials
-  # in the App Search dashboard to find the key associated with your account.
+  # The private API Key with write permissions.
+  # https://www.elastic.co/guide/en/app-search/current/authentication.html#authentication-api-keys
   config :api_key, :validate => :password, :required => true
 
   # Where to move the value from the `@timestamp` field.
@@ -46,57 +43,48 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
   # like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
   config :document_id, :validate => :string
 
-  # The path that is appended to the `url` parameter when connecting to a https://www.elastic.co/downloads/app-search
-  config :path, :validate => :string, :default => "/api/as/v1/"
-
-  ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/
+  ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/.freeze
 
-  public
   def register
-    @use_old_client = false
-    if @host.nil? && @url.nil?
-      raise ::LogStash::ConfigurationError.new("Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
-    elsif @host && @url
-      raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed ot Elastic Enterprise Search) or \"host\" (for SaaS).")
-    elsif @host && path_is_set?  # because path has a default value we need extra work to if the user set it
-      raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
-    elsif @host
-      @deprecation_logger.deprecated("Deprecated service usage, the `host` setting will be removed when Swiftype AppSearch service is shutdown")
-      @use_old_client = true
-      @client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
-    elsif @url
-      if path_is_set?
-        @deprecation_logger.deprecated("Deprecated service usage, the `path` setting will be removed when Swiftype AppSearch service is shutdown")
-        @use_old_client = true
-        @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
-      else
-        @client = Elastic::EnterpriseSearch::AppSearch::Client.new(:host => @url, :http_auth => @api_key.value, :external_url => @url)
-      end
-    end
-    check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
+    log_message = "The App Search product is deprecated and not supported from version 9 of the Elastic Stack. " +
+      "The Elastic App Search output plugin is deprecated and will only receive security updates and critical bug fixes. " +
+      "Please migrate to the Elastic Connector for continued support. " +
+      "For more details, please visit https://www.elastic.co/guide/en/search-ui/current/tutorials-elasticsearch.html"
+    deprecation_logger.deprecated log_message
+
+    @retry_disabled = false
+    @client = LogStash::PluginMixins::EnterpriseSearch::AppSearch::Client.new(client_options, params: params)
+    check_connection!
   rescue => e
     if e.message =~ /401/
-      raise ::LogStash::ConfigurationError.new("Failed to connect to App Search. Error: 401. Please check your credentials")
+      raise ::LogStash::ConfigurationError, "Failed to connect to App Search. Please check your credentials. Error: #{e.message}"
     elsif e.message =~ /404/
-      raise ::LogStash::ConfigurationError.new("Failed to connect to App Search. Error: 404. Please check if host '#{@host}' is correct and you've created an engine with name '#{@engine}'")
+      raise ::LogStash::ConfigurationError, "Failed to connect to App Search. Please check if url '#{@url}' is correct and you've created an engine with name '#{@engine}'. Error: #{e.message}"
     else
-      raise ::LogStash::ConfigurationError.new("Failed to connect to App Search. #{e.message}")
+      raise ::LogStash::ConfigurationError, "Failed to connect to App Search. Error: #{e.message}"
     end
   end
 
-  public
   def multi_receive(events)
     # because App Search has a limit of 100 documents per bulk
     events.each_slice(100) do |events|
       batch = format_batch(events)
       if @logger.trace?
-        @logger.trace("Sending bulk to App Search", :size => batch.size, :data => batch.inspect)
+        @logger.trace('Sending bulk to App Search', :size => batch.size, :data => batch.inspect)
       end
       index(batch)
     end
   end
 
   private
+
+  def client_options
+    options = { :host => @url, :http_auth => @api_key.value }
+    options[:logger] = @logger if @logger.debug?
+    options[:tracer] = @logger if @logger.trace?
+    options
+  end
+
   def format_batch(events)
     docs_for_engine = {}
     events.each do |event|
@@ -104,18 +92,18 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
       # we need to remove default fields that start with "@"
       # since Elastic App Search doesn't accept them
       if @timestamp_destination
-        doc[@timestamp_destination] = doc.delete("@timestamp")
+        doc[@timestamp_destination] = doc.delete('@timestamp')
       else # delete it
-        doc.delete("@timestamp")
+        doc.delete('@timestamp')
       end
       if @document_id
-        doc["id"] = event.sprintf(@document_id)
+        doc['id'] = event.sprintf(@document_id)
       end
-      doc.delete("@version")
+      doc.delete('@version')
       resolved_engine = event.sprintf(@engine)
       unless docs_for_engine[resolved_engine]
         if @logger.debug?
-          @logger.debug("Creating new engine segment in batch to send", :resolved_engine => resolved_engine)
+          @logger.debug('Creating new engine segment in batch to send', :resolved_engine => resolved_engine)
         end
         docs_for_engine[resolved_engine] = []
       end
@@ -124,21 +112,21 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
     docs_for_engine
   end
 
-  def index(batch)
-    batch.each do |resolved_engine, documents|
+  def index(docs_partitioned_by_engine)
+    docs_partitioned_by_engine.each do |resolved_engine, documents|
       begin
         if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
           raise "Cannot resolve engine field name #{@engine} from event"
         end
-        if connected_to_swiftype?
-          response = @client.index_documents(resolved_engine, documents)
-        else
-          response = @client.index_documents(resolved_engine, {:documents => documents})
-        end
+
+        response = @client.index_documents(resolved_engine, { :documents => documents })
         report(documents, response)
       rescue => e
-        @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
-                      :resolved_engine => resolved_engine, :backtrace => e.backtrace)
+        @logger.error('Failed to execute index operation.', :exception => e.class, :reason => e.message,
+                      :resolved_engine => resolved_engine, :backtrace => e.backtrace, :retry => !@retry_disabled)
+
+        raise e if @retry_disabled
+
         sleep(1)
         retry
       end
@@ -147,33 +135,17 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
 
   def report(documents, response)
     documents.each_with_index do |document, i|
-      if connected_to_swiftype?
-        errors = response[i]["errors"]
-      else
-        errors = response.body[i]["errors"]
-      end
+      errors = response.body[i]['errors']
       if errors.empty?
-        @logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
+        @logger.trace? && @logger.trace('Document was indexed with no errors', :document => document)
       else
-        @logger.warn("Document failed to index. Dropping..", :document => document, :errors => errors.to_a)
+        @logger.warn('Document failed to index. Dropping..', :document => document, :errors => errors.to_a)
       end
     end
   end
 
   def check_connection!
-    if connected_to_swiftype?
-      @client.get_engine(@engine)
-    else
-      res = @client.list_engines({:page_size => 1})
-      raise "Received HTTP error code #{res.status}" unless res.status == 200
-    end
-  end
-
-  def path_is_set?
-    original_params.key?("path")
-  end
-
-  def connected_to_swiftype?
-    @use_old_client
+    res = @client.list_engines({ 'page[size]': 1 })
+    raise "Received HTTP error code #{res.status}" unless res.status == 200
   end
 end

data/lib/logstash/outputs/elastic_workplace_search.rb

@@ -1,12 +1,18 @@
 # encoding: utf-8
-require "logstash/outputs/base"
-require "elastic-workplace-search"
+require 'logstash/outputs/base'
+require 'logstash/plugin_mixins/deprecation_logger_support'
+require 'logstash/plugin_mixins/enterprise_search/client'
+require 'logstash/plugin_mixins/enterprise_search/ssl_configs'
 
 class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
-  config_name "elastic_workplace_search"
+
+  include LogStash::PluginMixins::EnterpriseSearch::SSLConfigs
+  include LogStash::PluginMixins::DeprecationLoggerSupport
+
+  config_name 'elastic_workplace_search'
 
   # The value of the API endpoint in the form of a URL.
-  config :url, :validate => :string, :required => true
+  config :url, :validate => :string, :required => true, :default => Elastic::EnterpriseSearch::Utils::DEFAULT_HOST
 
   # The ID of the source you created in Workplace Search.
   # The `source` field supports
@@ -36,72 +42,104 @@ class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
   # like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
   config :document_id, :validate => :string
 
-  public
+  SOURCE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/.freeze
+
   def register
-    Elastic::WorkplaceSearch.endpoint = "#{@url.chomp('/')}/api/ws/v1/"
-    @client = Elastic::WorkplaceSearch::Client.new(:access_token => @access_token.value)
-    check_connection!
-  rescue Elastic::WorkplaceSearch::InvalidCredentials
-    raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 401. Please check your credentials")
-  rescue Elastic::WorkplaceSearch::NonExistentRecord
-    raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 404. Please check if url '#{@url}' is correct and you've created a source with ID '#{@source}'")
-  rescue => e
-    raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. #{e.message}")
+    log_message = "The Workplace Search product is deprecated and not supported from version 9 of the Elastic Stack. " +
+      "The Elastic Workplace Search output plugin is deprecated and will only receive security updates and critical bug fixes. " +
+      "Please migrate to the Elastic Connector for continued support. " +
+      "For more details, please visit https://www.elastic.co/guide/en/search-ui/current/tutorials-elasticsearch.html"
+    deprecation_logger.deprecated log_message
+
+    @retry_disabled = false
+    @client = LogStash::PluginMixins::EnterpriseSearch::WorkplaceSearch::Client.new(client_options, params: params)
+    begin
+      check_connection!
+    rescue => e
+      raise ::LogStash::ConfigurationError, "Failed to connect to Workplace Search. Error: #{e.message}"
+    end
   end
 
-  public
   def multi_receive(events)
     # because Workplace Search has a limit of 100 documents per bulk
     events.each_slice(100) do |events|
       batch = format_batch(events)
-      if @logger.trace?
-        @logger.trace("Sending bulk to Workplace Search", :size => batch.size, :data => batch.inspect)
-      end
+      @logger.trace('Sending bulk to Workplace Search', :size => batch.size, :data => batch.inspect) if @logger.trace?
       index(batch)
     end
   end
 
   private
+
+  def client_options
+    options = { :host => @url, :http_auth => @access_token.value }
+    options[:logger] = @logger if @logger.debug?
+    options[:tracer] = @logger if @logger.trace?
+    options
+  end
+
   def format_batch(events)
+    docs_per_source = {}
     events.map do |event|
       doc = event.to_hash
       # we need to remove default fields that start with "@"
       # since Elastic Workplace Search doesn't accept them
       if @timestamp_destination
-        doc[@timestamp_destination] = doc.delete("@timestamp")
+        doc[@timestamp_destination] = doc.delete('@timestamp')
       else # delete it
-        doc.delete("@timestamp")
+        doc.delete('@timestamp')
       end
-      if @document_id
-        doc["id"] = event.sprintf(@document_id)
+
+      doc['id'] = event.sprintf(@document_id) if @document_id
+      doc.delete('@version')
+
+      resolved_source = event.sprintf(@source)
+      unless docs_per_source[resolved_source]
+        @logger.debug('Creating new source segment in batch to send', resolved_source: resolved_source) if @logger.debug?
+        docs_per_source[resolved_source] = []
+      end
+      docs_per_source[resolved_source] << doc
+    end
+
+    docs_per_source
+  end
+
+  def index(docs_partitioned_by_source)
+    docs_partitioned_by_source.each do |resolved_source, documents|
+      begin
+        raise "Cannot resolve source field name #{@source} from event" unless resolved?(resolved_source)
+
+        response = @client.index_documents(resolved_source, { :documents => documents })
+        report(documents, response)
+      rescue => e
+        @logger.error('Failed to execute index operation.', :exception => e.class, :reason => e.message,
+                      :resolved_source => resolved_source, :backtrace => e.backtrace, :retry => !@retry_disabled)
+
+        raise e if @retry_disabled
+
+        sleep(1)
+        retry
       end
-      doc.delete("@version")
-      doc
     end
   end
 
-  def index(documents)
-    response = @client.index_documents(@source, documents)
-    report(documents, response)
-  rescue => e
-    @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message)
-    sleep(1)
-    retry
+  def resolved?(source)
+    !(source =~ SOURCE_WITH_SPRINTF_REGEX) && !(source =~ /^\s*$/)
   end
 
   def report(documents, response)
     documents.each_with_index do |document, i|
-      errors = response["results"][i]["errors"]
-      if errors.empty?
-        @logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
+      errors = response.body&.dig('results', i, 'errors')
+      if errors&.empty?
+        @logger.trace? && @logger.trace('Document was indexed with no errors', :document => document)
       else
-        @logger.warn("Document failed to index. Dropping..", :document => document, :errors => errors.to_a)
+        @logger.warn('Document failed to index. Dropping..', :document => document, :errors => errors.to_a)
       end
     end
   end
 
   def check_connection!
-    # This is the preferred way to check compatibility across different versions of Workplace Search service.
-    @client.index_documents(@source, {:documents => []})
+    res = @client.list_content_sources({ 'page[size]': 1 })
+    raise "Received HTTP error code #{res.status}" unless res.status == 200
   end
 end

data/lib/logstash/plugin_mixins/enterprise_search/client.rb

@@ -0,0 +1,35 @@
+require 'elastic-enterprise-search'
+require 'logstash/plugin_mixins/enterprise_search/manticore_transport'
+
+module LogStash::PluginMixins::EnterpriseSearch
+
+  module AppSearch
+    # App Search client for Enterprise Search.
+    # This client extends Elastic::EnterpriseSearch::AppSearch::Client but overrides #transport to use Manticore.
+    class Client < Elastic::EnterpriseSearch::AppSearch::Client
+      attr_reader :params
+
+      include LogStash::PluginMixins::EnterpriseSearch::ManticoreTransport
+
+      def initialize(options, params: {})
+        @params = params
+        super options
+      end
+    end
+  end
+
+  module WorkplaceSearch
+    # Workplace Search client for Enterprise Search.
+    # This client extends Elastic::EnterpriseSearch::WorkplaceSearch::Client but overrides #transport to use Manticore.
+    class Client < Elastic::EnterpriseSearch::WorkplaceSearch::Client
+      attr_reader :params
+
+      include LogStash::PluginMixins::EnterpriseSearch::ManticoreTransport
+
+      def initialize(options, params: {})
+        @params = params
+        super options
+      end
+    end
+  end
+end

data/lib/logstash/plugin_mixins/enterprise_search/manticore_transport.rb

@@ -0,0 +1,84 @@
+module LogStash::PluginMixins::EnterpriseSearch
+  # This module is meant to be included in EnterpriseSearch::Clients subclasses and overrides
+  # the default #transport method, changing the HTTP client to Manticore.
+  # It also provides common Manticore configurations such as :ssl.
+  module ManticoreTransport
+    def self.included(base)
+      if eps_version_7?
+        require 'elasticsearch/transport/transport/http/manticore'
+      else
+        require 'elastic/transport/transport/http/manticore'
+      end
+
+      raise ArgumentError, "`#{base}` must inherit Elastic::EnterpriseSearch::Client" unless base < Elastic::EnterpriseSearch::Client
+      raise ArgumentError, "`#{base}` must respond to :params" unless base.instance_methods.include? :params
+    end
+
+    # overrides Elastic::EnterpriseSearch::Client#transport
+    def transport
+      @options[:transport] || transport_klass.new(
+        host: host,
+        log: log,
+        logger: logger,
+        request_timeout: overall_timeout,
+        adapter: @options[:adapter],
+        transport_options: {
+          request: { open_timeout: open_timeout }
+        },
+        transport_class: manticore_transport_klass,
+        ssl: build_ssl_config,
+        enable_meta_header: @options[:enable_meta_header] || true,
+        trace: !@options[:tracer].nil?,
+        tracer: @options[:tracer]
+      )
+    end
+
+    def self.eps_version_7?
+      Elastic::EnterpriseSearch::VERSION.start_with?('7')
+    end
+
+    private
+
+    def manticore_transport_klass
+      ManticoreTransport.eps_version_7? ? Elasticsearch::Transport::Transport::HTTP::Manticore : Elastic::Transport::Transport::HTTP::Manticore
+    end
+
+    def transport_klass
+      case Elasticsearch::Transport::VERSION
+      when /^7\.1[123]/
+        Elasticsearch::Client
+      else
+        Elasticsearch::Transport::Client
+      end
+    end
+
+    def build_transport_options
+      { request: { open_timeout: open_timeout } }
+    end
+
+    def build_ssl_config
+      ssl_certificate_authorities, ssl_truststore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path')
+      if ssl_certificate_authorities && ssl_truststore_path
+        raise ::LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities" or "ssl_truststore_path" when configuring the CA certificate'
+      end
+
+      ssl_options = {}
+      if ssl_certificate_authorities&.any?
+        raise ::LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin' if ssl_certificate_authorities.size > 1
+
+        ssl_options[:ca_file] = ssl_certificate_authorities.first
+      end
+
+      if ssl_truststore_path
+        ssl_options[:truststore] = ssl_truststore_path
+        ssl_options[:truststore_type] = params['ssl_truststore_type'] if params.include?('ssl_truststore_type')
+        ssl_options[:truststore_password] = params['ssl_truststore_password'].value if params.include?('ssl_truststore_password')
+      end
+
+      ssl_options[:verify] = params['ssl_verification_mode'] == 'full' ? :strict : :disable
+      ssl_options[:cipher_suites] = params['ssl_cipher_suites'] if params.include?('ssl_cipher_suites')
+      ssl_options[:protocols] = params['ssl_supported_protocols'] if params['ssl_supported_protocols']&.any?
+      ssl_options
+    end
+  end
+end

data/lib/logstash/plugin_mixins/enterprise_search/ssl_configs.rb

@@ -0,0 +1,32 @@
+module LogStash::PluginMixins::EnterpriseSearch
+  # This module defines common SSL options that can be reused by the app and workplace search plugins.
+  module SSLConfigs
+    def self.included(base)
+      # SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary
+      base.config :ssl_certificate_authorities, :validate => :path, :list => true
+
+      # The JKS truststore to validate the server's certificate.
+      # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
+      base.config :ssl_truststore_path, :validate => :path
+
+      # Set the truststore password
+      base.config :ssl_truststore_password, :validate => :password
+
+      # The format of the truststore file. It must be either jks or pkcs12
+      base.config :ssl_truststore_type, :validate => %w[pkcs12 jks]
+
+      # Options to verify the server's certificate.
+      # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
+      # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
+      # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
+      base.config :ssl_verification_mode, :validate => %w[full none], :default => 'full'
+
+      # Supported protocols with versions.
+      base.config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true
+
+      # The list of cipher suites to use, listed by priorities.
+      # Supported cipher suites vary depending on which version of Java is used.
+      base.config :ssl_cipher_suites, :validate => :string, :list => true
+    end
+  end
+end

data/logstash-integration-elastic_enterprise_search.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-integration-elastic_enterprise_search'
-  s.version         = '2.2.1'
+  s.version         = '3.0.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Elastic Enterprise Search - output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -24,11 +24,10 @@ Gem::Specification.new do |s|
   }
 
   # Gem dependencies
+  s.add_runtime_dependency "manticore", '~> 0.8'
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency "logstash-codec-plain"
-  s.add_runtime_dependency "elastic-app-search", '~>7.8.0'
-  s.add_runtime_dependency "elastic-enterprise-search", '~>7.16.0'
-  s.add_runtime_dependency "elastic-workplace-search", '~>0.4.1'
+  s.add_runtime_dependency 'elastic-enterprise-search', '>= 7.16', '< 9'
   s.add_runtime_dependency "logstash-mixin-deprecation_logger_support", '~>1.0'
   s.add_development_dependency "logstash-devutils"
 end

data/spec/fixtures/certificates/generate.sh

@@ -0,0 +1,10 @@
+# Warning: do not use the certificates produced by this tool in production. This is for testing purposes only
+
+openssl genrsa 4096 | openssl pkcs8 -topk8 -nocrypt -out root_ca.key
+openssl req -sha256 -x509 -newkey rsa:4096 -nodes -key root_ca.key -sha256 -days 365 -out root_ca.crt -subj "/C=ES/ST=The Internet/L=The Internet/O=Logstash CA/OU=Logstash/CN=enterprise_search"
+openssl req -sha256 -x509 -newkey rsa:4096 -nodes -key root_ca.key -sha256 -days 365 -out root_untrusted_ca.crt -subj "/C=ES/ST=The Darknet/L=The Darknet/O=Logstash CA/OU=Logstash/CN=127.0.0.1"
+openssl pkcs12 -export -in root_ca.crt -inkey root_ca.key -out root_keystore.p12 -password pass:changeme -name ent-search
+keytool -importkeystore -srckeystore root_keystore.p12 -destkeystore root_keystore.jks -srcstoretype PKCS12 -deststoretype jks -srcstorepass changeme -deststorepass changeme -srcalias ent-search -destalias ent-search -srckeypass changeme -destkeypass changeme
+
+rm -rf root_keystore.p12
+rm -rf *.csr

data/spec/fixtures/certificates/root_ca.crt

@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF4zCCA8ugAwIBAgIUQ9JjwQESSVBfFxIEw50tni49yaUwDQYJKoZIhvcNAQEL
+BQAwgYAxCzAJBgNVBAYTAkVTMRUwEwYDVQQIDAxUaGUgSW50ZXJuZXQxFTATBgNV
+BAcMDFRoZSBJbnRlcm5ldDEUMBIGA1UECgwLTG9nc3Rhc2ggQ0ExETAPBgNVBAsM
+CExvZ3N0YXNoMRowGAYDVQQDDBFlbnRlcnByaXNlX3NlYXJjaDAeFw0yMzEwMTYx
+NDUwNDZaFw0yNDEwMTUxNDUwNDZaMIGAMQswCQYDVQQGEwJFUzEVMBMGA1UECAwM
+VGhlIEludGVybmV0MRUwEwYDVQQHDAxUaGUgSW50ZXJuZXQxFDASBgNVBAoMC0xv
+Z3N0YXNoIENBMREwDwYDVQQLDAhMb2dzdGFzaDEaMBgGA1UEAwwRZW50ZXJwcmlz
+ZV9zZWFyY2gwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDnyKWaGkRr
+3N8z7lG7cD49FZlk8p26AVG213xKZaj4VlfJfcQZQ8AoGXzOFI5ow6qhnChglT7t
+yngZU16TZdY0Iosh1ZFvt7S3N00tamYQdAH1IUQPvygHPZvs66Ik3G/+NOFeGVQK
+YFkUiuxUgGJ13Q+lFMRSshDay0rf8ekRnFE73roHBveo8Pr7TWDB/eacphQe+RPE
+VB4VYILqv/J+PByzYxidFMdhY5rlMYyRk/SOsOqUcXVLw94yo2D42OIcNMSkpdpx
+xTH42iInWeGgyWkMCDZV9aUWNcXdZ+zRLO0IZD3LxRJ0c/QSFjwrfIVoSkNQyMq7
+b9HPimsc3Ud+YsRQiYuRsMIeAVFNrIZxbbC61o///hWvM2loTai8uOhdxj72l2pC
+q3VzI+i3rkX0x0/oFx00h686batF+bPHwBUuJpzf+KMJPCiV5Z916TnjHA1OrtBd
+6WH+ckFmL+6TwjzLk9mg9iYWVFnky9+mG6c2svgjn05KA++2r2rVsXQPx8DwppgD
+dEk570ciu0EhP94oZlw0smtf/uhp/wVym9Z1IOjo3i19DKeF+DzPTD2ylSi/43ub
+ICeKGG70ZT3eP2Nes58z/GXhxHFYHLUJcg6KjvaPWiv6KciGWwtR/XuT2aoShlpp
+sZv6u5ckPWxYvhZK0+07yxyoCrLgGGO1XQIDAQABo1MwUTAdBgNVHQ4EFgQU+pka
+cSbOMgRJk0Gi4ChZrWo8vrMwHwYDVR0jBBgwFoAU+pkacSbOMgRJk0Gi4ChZrWo8
+vrMwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAeQHpkts7hzod
+NzawMbawu6zPx3d6hP1OaUqU4sWa5OHu0eLebDLn4HQ9q6FJjuFREGHjktfKJOaf
+lpflQrCioHcmicQ9lTHqm7SyjokteYCbQMJOLpn41TzHUVlufxqTgPGf3QjjcrEh
+SaplxNFZxFXJO383aZfELfLojLtE4bZK8olx7l2mnPSVbhe/TJzpIoFvICOfw/H4
+pdE4SuLJGblaMBV57guRziI+PBqDulUeq92/ZWE5kDO32itgxEObNzbehiUcKNS8
+LeCiniGXvvGAUaWnekueMJeYm65cJZzp5uLULSW5aL4mCBuyJskpmHb3iYZvthEy
+vJwNZZc10/i7ogfUUMQUtmvYTPcmG50Mkjt4g7wToDzHna8avE2XKx/0zyyQi+GA
+/9NFpKTgNbmj/FKnswGGBn25jOUyhEhbhIIapNL63ZLfS3t1SCFc7ocXgTHY2jF4
+JFE9pOieKsAjSNbLcEMCfrI+TWAJRCarkLmzvo/IsyXrMJZzW4lxQmzskfxV0/ta
+y1wbhwD9baEJBoJV0HJUYmUKv0o4IOdPWUtz4muYo4mBGOUMXy2BKFh/Cv6Dx2OV
+VHCW1SMgyzGgpF4w+Jy/+hxNRcJPfB1bhnklR4mctWPR6UuMLn2nhWhttoO29UGt
+qk9INcfDhzlxwVIqYAoLXTV/ru3mTZI=
+-----END CERTIFICATE-----