logstash-filter-elasticsearch 4.0.0 → 4.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 015a98dbd36122dd3fc4c74da5744a31f6182f67023f802189cc73e837d5ba7a
-  data.tar.gz: 8f1c3a79c0af3fc4154501d16bf775e01b6aa7575627852cb18954a0dd952d91
+  metadata.gz: 1ca5815c2b1127152e31b4ca9e0192021bfb0b0d9d2807ca090c2123feea2bb8
+  data.tar.gz: b3983e01f4e73076e48f3d793f2061cb3210490b09394352146b65377f946137
 SHA512:
-  metadata.gz: fc3971541568e34c0ef644c250243fefcb48c778b2471f81f5e87a1b59535a6f5ccc2a2c13d57f5123e0577efb80962b85f8096d3d6f70e0721df4f716d705e1
-  data.tar.gz: 5fe74aa6d8179e6dd3d9c69a6d0c81f284035bd82343543132f129be7bfb3355e01678eddbd051d9d4c360e9704f8a234737096e757f710b395b5d366f57936f
+  metadata.gz: 65822b043de7055e2422810c4b03662c45fb15d58f45edadd677a220287d37a8f8c7d1382e3b1dc04e00ac51e261a1e5f7e11c06b4d57481218be9da2f498534
+  data.tar.gz: 8738742640bf8729297f556154d6ef11878e99b55bfe1193da459f2f83cf255d5e5973537bc6e97e5d32021d1976f6aa4c284081a98f5b7a84c1073dfa039cc2

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.1.1
+  - Add elastic-transport client support used in elasticsearch-ruby 8.x [#191](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/191)
+
+## 4.1.0
+ - Added support for custom headers [#188](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/188)
+
 ## 4.0.0
   - SSL settings that were marked deprecated in version `3.15.0` are now marked obsolete, and will prevent the plugin from starting.
   - These settings are:

data/docs/index.asciidoc

@@ -134,6 +134,7 @@ NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated se
 | <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-enable_sort>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
@@ -230,6 +231,16 @@ Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 For more info, check out the
 {logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
 
+
+[id="plugins-{type}s-{plugin}-custom_headers"]
+===== `custom_headers`
+
+  * Value type is <<hash,hash>>
+  * Default value is empty
+
+Pass a set of key value pairs as the headers sent in each request to Elasticsearch.
+These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers.
+
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
 

data/lib/logstash/filters/elasticsearch/client.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "elasticsearch"
 require "base64"
-require "elasticsearch/transport/transport/http/manticore"
 
 
 module LogStash
@@ -9,6 +8,7 @@ module LogStash
     class ElasticsearchClient
 
       attr_reader :client
+      attr_reader :es_transport_client_type
 
       BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
       DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
@@ -20,6 +20,8 @@ module LogStash
         api_key = options.fetch(:api_key, nil)
         proxy = options.fetch(:proxy, nil)
         user_agent = options[:user_agent]
+        custom_headers = options[:custom_headers]
+
 
         transport_options = { }
         transport_options[:headers] = options.fetch(:serverless, false) ?  DEFAULT_EAV_HEADER.dup : {}
@@ -27,6 +29,7 @@ module LogStash
         transport_options[:headers].merge!(setup_api_key(api_key))
         transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
         transport_options[:headers].merge!(INTERNAL_ORIGIN_HEADER)
+        transport_options[:headers].merge!(custom_headers) unless custom_headers.empty?
 
         transport_options[:pool_max] = 1000
         transport_options[:pool_max_per_route] = 100
@@ -41,7 +44,7 @@ module LogStash
 
         client_options = {
                        hosts: hosts,
-             transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore,
+             transport_class: get_transport_client_class,
            transport_options: transport_options,
                          ssl: ssl_options,
             retry_on_failure: options[:retry_on_failure],
@@ -95,6 +98,20 @@ module LogStash
         token = ::Base64.strict_encode64(api_key.value)
         { 'Authorization' => "ApiKey #{token}" }
       end
+
+      def get_transport_client_class
+        # LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport`
+        # And now `elasticsearch-transport` is old, instead we have `elastic-transport`.
+        # LS-core updated `elasticsearch` > 8: https://github.com/elastic/logstash/pull/17161
+        # Following source bits are for the compatibility to support both `elasticsearch-transport` and `elastic-transport` gems
+        require "elasticsearch/transport/transport/http/manticore"
+        es_transport_client_type = "elasticsearch_transport"
+        ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      rescue ::LoadError
+        require "elastic/transport/transport/http/manticore"
+        es_transport_client_type = "elastic_transport"
+        ::Elastic::Transport::Transport::HTTP::Manticore
+      end
     end
   end
 end

data/lib/logstash/filters/elasticsearch.rb

@@ -6,7 +6,6 @@ require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "monitor"
 
 require_relative "elasticsearch/client"
-require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config_name "elasticsearch"
@@ -32,6 +31,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Array of fields to copy from old event (found via elasticsearch) into new event
   config :fields, :validate => :array, :default => {}
 
+  # Custom headers for Elasticsearch requests
+  config :custom_headers, :validate => :hash, :default => {}
+
   # Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
   config :docinfo_fields, :validate => :hash, :default => {}
 
@@ -171,6 +173,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
     test_connection!
     setup_serverless
+    if get_client.es_transport_client_type == "elasticsearch_transport"
+      require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
+    end
   end # def register
 
   def filter(event)
@@ -260,7 +265,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       :ssl => client_ssl_options,
       :retry_on_failure => @retry_on_failure,
       :retry_on_status => @retry_on_status,
-      :user_agent => prepare_user_agent
+      :user_agent => prepare_user_agent,
+      :custom_headers => @custom_headers
     }
   end
 

data/logstash-filter-elasticsearch.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '4.0.0'
+  s.version         = '4.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
-  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.homepage        = "https://elastic.co/logstash"
   s.require_paths = ["lib"]
 
   # Files
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9", '< 9'
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
   s.add_development_dependency 'cabin', ['~> 0.6']

data/spec/filters/elasticsearch_spec.rb

@@ -60,9 +60,18 @@ describe LogStash::Filters::Elasticsearch do
         allow(plugin).to receive(:get_client).and_return(filter_client)
         allow(filter_client).to receive(:serverless?).and_return(true)
         allow(filter_client).to receive(:client).and_return(es_client)
-        allow(es_client).to receive(:info).with(a_hash_including(:headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)).and_raise(
-          Elasticsearch::Transport::Transport::Errors::BadRequest.new
-        )
+
+        if elastic_ruby_v8_client_available?
+          allow(es_client).to receive(:info)
+                                .with(a_hash_including(
+                                        :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER))
+                                .and_raise(Elastic::Transport::Transport::Errors::BadRequest.new)
+        else
+          allow(es_client).to receive(:info)
+                                .with(a_hash_including(
+                                        :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER))
+                                .and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest.new)
+        end
       end
 
       it "raises an exception when Elastic Api Version is not supported" do
@@ -103,6 +112,11 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+      if elastic_ruby_v8_client_available?
+        allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
+      else
+        allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
+      end
       allow(client).to receive(:search).and_return(response)
       allow(plugin).to receive(:test_connection!)
       allow(plugin).to receive(:setup_serverless)
@@ -333,6 +347,35 @@ describe LogStash::Filters::Elasticsearch do
       end
     end
 
+    context "with custom headers" do
+      let(:config) do
+        {
+          "query" => "*",
+          "custom_headers" => { "Custom-Header-1" => "Custom Value 1", "Custom-Header-2" => "Custom Value 2" }
+        }
+      end
+
+      let(:plugin) { LogStash::Filters::Elasticsearch.new(config) }
+      let(:client_double) { double("client") }
+      let(:transport_double) { double("transport", options: { transport_options: { headers: config["custom_headers"] } }) }
+
+      before do
+        allow(plugin).to receive(:get_client).and_return(client_double)
+        if elastic_ruby_v8_client_available?
+          allow(client_double).to receive(:es_transport_client_type).and_return('elastic_transport')
+        else
+          allow(client_double).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
+        end
+        allow(client_double).to receive(:client).and_return(transport_double)
+      end
+
+      it "sets custom headers" do
+        plugin.register
+        client = plugin.send(:get_client).client
+        expect(client.options[:transport_options][:headers]).to match(hash_including(config["custom_headers"]))
+      end
+    end
+
     context "if query is on nested field" do
       let(:config) do
         {
@@ -482,7 +525,12 @@ describe LogStash::Filters::Elasticsearch do
       # this spec is a safeguard to trigger an assessment of thread-safety should
       # we choose a different transport adapter in the future.
       transport_class = extract_transport(client).options.fetch(:transport_class)
-      expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      if elastic_ruby_v8_client_available?
+        allow(client).to receive(:es_transport_client_type).and_return("elastic_transport")
+        expect(transport_class).to equal ::Elastic::Transport::Transport::HTTP::Manticore
+      else
+        expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      end
     end
 
     it 'uses a client with sufficient connection pool size' do
@@ -797,6 +845,11 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+      if elastic_ruby_v8_client_available?
+        allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
+      else
+        allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
+      end
       allow(plugin).to receive(:test_connection!)
       allow(plugin).to receive(:setup_serverless)
       plugin.register
@@ -811,11 +864,19 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
-  # @note can be removed once gem depends on elasticsearch >= 6.x
-  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
+  def extract_transport(client)
+    # on 7x: client.transport.transport
+    # on >=8.x: client.transport
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
+  def elastic_ruby_v8_client_available?
+    Elasticsearch::Transport
+    false
+  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
+    true
+  end
+
   class MockResponse
     attr_reader :code, :headers
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.1.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-10 00:00:00.000000000 Z
+date: 2025-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -36,6 +36,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.14.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
   name: elasticsearch
   type: :runtime
   prerelease: false
@@ -44,6 +47,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.14.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -160,7 +166,7 @@ files:
 - spec/filters/fixtures/test_certs/ls.der.sha256
 - spec/filters/fixtures/test_certs/ls.key
 - spec/filters/integration/elasticsearch_spec.rb
-homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+homepage: https://elastic.co/logstash
 licenses:
 - Apache License (2.0)
 metadata: