logstash-filter-elasticsearch 3.15.3 → 3.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e85d9b1122e7e095666cdc3bfa1dd5fbd3ef62ce5cf15d667ffa92932e5748e
-  data.tar.gz: cbcf76ec5d048bb18f467520f4a646a08e5ea39a8d44ace7a3322e7fea0eee06
+  metadata.gz: c214ce4f38ac730daec06cafc6c71630b1d6c3c9aacaf2fcbbf3457c055401f1
+  data.tar.gz: 2139bde2dffc838b33ae816325bed9172d78cbb66125f6c204b9e8dd802a9879
 SHA512:
-  metadata.gz: 6bc45e8790eede5047013506fe8b9f8ba97ad08365f6ab99d5fd7ca120e051f916301054e23932d068823ba8f0904757de9bc0bb255d13b1967b339c600e4011
-  data.tar.gz: '07842227fb3c356f08012193925fe1b5ae055ead761600c029e0ea4a788b97744113e35ebf97c05e223270e14017bf1d10b66a0ca038a810ca5054c1c2ff8389'
+  metadata.gz: 7de34c594ee9d49c567cb70809fc09ebe70f19def9c14e8898a080a7eaac0e889ab0e714a9cae0341bc200e661a5cbf2faca97eb95aab18423387eff24eae88b
+  data.tar.gz: c0cc4308ef74c309a326d47c0c3901811b3b8c0f5f0e3619949cc9b7cc48cb81d078d65391f3020b221f99774a402d90d70d5248094e26f64859248d651a7e61

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.16.0
+  - Added request header `Elastic-Api-Version` for serverless [#174](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/174)
+
 ## 3.15.3
   - Fixes a memory leak that occurs when a pipeline containing this filter terminates, which could become significant if the pipeline is cycled repeatedly [#173](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/173)
 

data/lib/logstash/filters/elasticsearch/client.rb

@@ -10,6 +10,9 @@ module LogStash
 
       attr_reader :client
 
+      BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
+      DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
+
       def initialize(logger, hosts, options = {})
         user = options.fetch(:user, nil)
         password = options.fetch(:password, nil)
@@ -17,7 +20,8 @@ module LogStash
         proxy = options.fetch(:proxy, nil)
         user_agent = options[:user_agent]
 
-        transport_options = {:headers => {}}
+        transport_options = { }
+        transport_options[:headers] = options.fetch(:serverless, false) ?  DEFAULT_EAV_HEADER.dup : {}
         transport_options[:headers].merge!(setup_basic_auth(user, password))
         transport_options[:headers].merge!(setup_api_key(api_key))
         transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
@@ -46,10 +50,22 @@ module LogStash
         @client = ::Elasticsearch::Client.new(client_options)
       end
 
-      def search(params)
+      def search(params={})
         @client.search(params)
       end
 
+      def info
+        @client.info
+      end
+
+      def build_flavor
+        @build_flavor ||= info&.dig('version', 'build_flavor')
+      end
+
+      def serverless?
+        @is_serverless ||= (build_flavor == BUILD_FLAVOR_SERVERLESS)
+      end
+
       private
 
       def setup_hosts(hosts, ssl_enabled)

data/lib/logstash/filters/elasticsearch.rb

@@ -179,6 +179,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
 
     test_connection!
+    setup_serverless
   end # def register
 
   def filter(event)
@@ -260,14 +261,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   private
 
   def client_options
-    {
+    @client_options ||= {
       :user => @user,
       :password => @password,
       :api_key => @api_key,
       :proxy => @proxy,
       :ssl => client_ssl_options,
       :retry_on_failure => @retry_on_failure,
-      :retry_on_status => @retry_on_status
+      :retry_on_status => @retry_on_status,
+      :user_agent => prepare_user_agent
     }
   end
 
@@ -344,11 +346,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   def new_client
     # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
     # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
-    opts = client_options
-
-    opts[:user_agent] = prepare_user_agent
-
-    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, opts)
+    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, client_options)
   end
 
   def get_client
@@ -478,6 +476,17 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
   end
 
+  def setup_serverless
+    if get_client.serverless?
+      @client_options[:serverless] = true
+      @shared_client = new_client
+      get_client.info
+    end
+  rescue => e
+    @logger.error("Failed to retrieve Elasticsearch info", message: e.message, exception: e.class, backtrace: e.backtrace)
+    raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
+  end
+
   def setup_ssl_params!
     @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
       normalize.with_deprecated_alias(:ssl)

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.15.3'
+  s.version         = '3.16.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 7.14.0" # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'

data/spec/filters/elasticsearch_spec.rb

@@ -22,6 +22,7 @@ describe LogStash::Filters::Elasticsearch do
 
       before do
         allow(plugin).to receive(:test_connection!)
+        allow(plugin).to receive(:setup_serverless)
       end
       
       it "should not raise an exception" do
@@ -49,6 +50,26 @@ describe LogStash::Filters::Elasticsearch do
       end
     end
 
+    context "against serverless Elasticsearch" do
+      let(:config) { { "query" => "*" } }
+      let(:filter_client) { double("filter_client") }
+      let(:es_client) { double("es_client") }
+
+      before do
+        allow(plugin).to receive(:test_connection!)
+        allow(plugin).to receive(:get_client).and_return(filter_client)
+        allow(filter_client).to receive(:serverless?).and_return(true)
+        allow(filter_client).to receive(:client).and_return(es_client)
+        allow(es_client).to receive(:info).with(a_hash_including(:headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)).and_raise(
+          Elasticsearch::Transport::Transport::Errors::BadRequest.new
+        )
+      end
+
+      it "raises an exception when Elastic Api Version is not supported" do
+        expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
     context "query settings" do
       it "raise an exception when query and query_template are empty" do
         plugin = described_class.new({})
@@ -84,6 +105,7 @@ describe LogStash::Filters::Elasticsearch do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
       allow(client).to receive(:search).and_return(response)
       allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
       plugin.register
     end
 
@@ -423,7 +445,9 @@ describe LogStash::Filters::Elasticsearch do
       let(:plugin) { described_class.new(config) }
       let(:event)  { LogStash::Event.new({}) }
 
-      it "client should sent the expect user-agent" do
+      # elasticsearch-ruby 7.17.9 initialize two user agent headers, `user-agent` and `User-Agent`
+      # hence, fail this header size test case
+      xit "client should sent the expect user-agent" do
         plugin.register
 
         request = webserver.wait_receive_request
@@ -445,6 +469,7 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
     end
 
     after(:each) do
@@ -619,6 +644,47 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
+  describe "Elastic Api Header" do
+    let(:config) { {"query" => "*"} }
+    let(:plugin) { described_class.new(config) }
+    let(:headers) {{'x-elastic-product' => 'Elasticsearch'}}
+    let(:cluster_info) { {"version" => {"number" => "8.10.0", "build_flavor" => build_flavor}, "tagline" => "You Know, for Search"} }
+    let(:mock_resp) { MockResponse.new(200, cluster_info, headers) }
+
+    before do
+      expect(plugin).to receive(:test_connection!)
+    end
+
+    context "serverless" do
+      let(:build_flavor) { "serverless" }
+
+      before do
+        allow_any_instance_of(Elasticsearch::Client).to receive(:perform_request).with(any_args).and_return(mock_resp)
+      end
+
+      it 'propagates header to es client' do
+        plugin.register
+        client = plugin.send(:get_client).client
+        expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("Elastic-Api-Version" => "2023-10-31")
+      end
+    end
+
+    context "stateful" do
+      let(:build_flavor) { "default" }
+
+      before do
+        expect_any_instance_of(Elasticsearch::Client).to receive(:perform_request).with(any_args).and_return(mock_resp)
+      end
+
+      it 'does not propagate header to es client' do
+        plugin.register
+        client = plugin.send(:get_client).client
+        expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_not_including("Elastic-Api-Version" => "2023-10-31")
+      end
+    end
+
+  end
+
   describe "ca_trusted_fingerprint" do
     let(:ca_trusted_fingerprint) { SecureRandom.hex(32) }
     let(:config) { {"ssl_enabled" => true, "ca_trusted_fingerprint" => ca_trusted_fingerprint, "query" => "*"}}
@@ -627,7 +693,10 @@ describe LogStash::Filters::Elasticsearch do
 
     if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
       context 'the generated trust_strategy' do
-        before(:each) { allow(plugin).to receive(:test_connection!) }
+        before(:each) do
+          allow(plugin).to receive(:test_connection!)
+          allow(plugin).to receive(:setup_serverless)
+        end
 
         it 'is passed to the Manticore client' do
           expect(Manticore::Client).to receive(:new)
@@ -666,7 +735,10 @@ describe LogStash::Filters::Elasticsearch do
 
     subject(:plugin) { described_class.new(config) }
 
-    before(:each) { allow(plugin).to receive(:test_connection!) }
+    before(:each) do
+      allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
+    end
 
     it 'is passed to the Manticore client' do
       expect(Manticore::Client).to receive(:new)
@@ -694,7 +766,10 @@ describe LogStash::Filters::Elasticsearch do
     let(:config) { {"query" => "*"} }
     let(:plugin) { described_class.new(config) }
 
-    before { allow(plugin).to receive(:test_connection!) }
+    before do
+      allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
+    end
 
     it "should set localhost:9200 as hosts" do
       plugin.register
@@ -719,6 +794,7 @@ describe LogStash::Filters::Elasticsearch do
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
       allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
       plugin.register
     end
 
@@ -736,4 +812,21 @@ describe LogStash::Filters::Elasticsearch do
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
+  class MockResponse
+    attr_reader :code, :headers
+
+    def initialize(code = 200, body = nil, headers = {})
+      @code = code
+      @body = body
+      @headers = headers
+    end
+
+    def body
+      @body
+    end
+
+    def status
+      @code
+    end
+  end
 end

data/spec/filters/elasticsearch_ssl_spec.rb

@@ -15,6 +15,8 @@ describe "SSL options" do
   before do
     allow(es_client_double).to receive(:close)
     allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object)
+    allow(es_client_double).to receive(:info).with(any_args).and_return({"version" => {"number" => "7.5.0", "build_flavor" => "default"},
+                                                                         "tagline" => "You Know, for Search"})
     allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double)
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.15.3
+  version: 3.16.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-08 00:00:00.000000000 Z
+date: 2023-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -35,7 +35,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.14.9
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.14.9
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: