logstash-filter-elasticsearch 3.15.2 → 3.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64700ddd93547ed4e08abfb73028ab17f7dc6bd6d591840c622b5f7e24b3d5c3
-  data.tar.gz: 9cdf64fc9afe9a2a66c453850d2b4c0d910af5251567799bfd6612737045ba50
+  metadata.gz: c214ce4f38ac730daec06cafc6c71630b1d6c3c9aacaf2fcbbf3457c055401f1
+  data.tar.gz: 2139bde2dffc838b33ae816325bed9172d78cbb66125f6c204b9e8dd802a9879
 SHA512:
-  metadata.gz: 54c710f94a363bb2f8c9b1ecf450ec3397b50792fb7d4c2035a2ee7c70b347c7a41a4c8d7959bcd0eb8403a5df786bdfbebb5bb600f54a4668d6f4905d429054
-  data.tar.gz: 4f788bb3592c42fc2003e22c94c49d53272063ec2fadbe85f49e3653ba7bdfbbf81024cb34d8e81cec7f7eaf9968fc2be8fd704da5678d34eae60a3fd47d184f
+  metadata.gz: 7de34c594ee9d49c567cb70809fc09ebe70f19def9c14e8898a080a7eaac0e889ab0e714a9cae0341bc200e661a5cbf2faca97eb95aab18423387eff24eae88b
+  data.tar.gz: c0cc4308ef74c309a326d47c0c3901811b3b8c0f5f0e3619949cc9b7cc48cb81d078d65391f3020b221f99774a402d90d70d5248094e26f64859248d651a7e61

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.16.0
+  - Added request header `Elastic-Api-Version` for serverless [#174](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/174)
+
+## 3.15.3
+  - Fixes a memory leak that occurs when a pipeline containing this filter terminates, which could become significant if the pipeline is cycled repeatedly [#173](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/173)
+
 ## 3.15.2
   - Added checking for `query` and `query_template`. [#171](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/171)
   

data/lib/logstash/filters/elasticsearch/client.rb

@@ -10,6 +10,9 @@ module LogStash
 
       attr_reader :client
 
+      BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
+      DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
+
       def initialize(logger, hosts, options = {})
         user = options.fetch(:user, nil)
         password = options.fetch(:password, nil)
@@ -17,11 +20,15 @@ module LogStash
         proxy = options.fetch(:proxy, nil)
         user_agent = options[:user_agent]
 
-        transport_options = {:headers => {}}
+        transport_options = { }
+        transport_options[:headers] = options.fetch(:serverless, false) ?  DEFAULT_EAV_HEADER.dup : {}
         transport_options[:headers].merge!(setup_basic_auth(user, password))
         transport_options[:headers].merge!(setup_api_key(api_key))
         transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
 
+        transport_options[:pool_max] = 1000
+        transport_options[:pool_max_per_route] = 100
+
         logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
         transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')
 
@@ -43,10 +50,22 @@ module LogStash
         @client = ::Elasticsearch::Client.new(client_options)
       end
 
-      def search(params)
+      def search(params={})
         @client.search(params)
       end
 
+      def info
+        @client.info
+      end
+
+      def build_flavor
+        @build_flavor ||= info&.dig('version', 'build_flavor')
+      end
+
+      def serverless?
+        @is_serverless ||= (build_flavor == BUILD_FLAVOR_SERVERLESS)
+      end
+
       private
 
       def setup_hosts(hosts, ssl_enabled)

data/lib/logstash/filters/elasticsearch.rb

@@ -4,6 +4,7 @@ require "logstash/namespace"
 require "logstash/json"
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "logstash/plugin_mixins/normalize_config_support"
+require "monitor"
 
 require_relative "elasticsearch/client"
 require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
@@ -139,7 +140,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   include LogStash::PluginMixins::NormalizeConfigSupport
 
-  attr_reader :clients_pool
+  include MonitorMixin
+  attr_reader :shared_client
 
   ##
   # @override to handle proxy => '' as if none was set
@@ -159,8 +161,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def register
-    @clients_pool = java.util.concurrent.ConcurrentHashMap.new
-
     #Load query if it exists
     if @query_template
       if File.zero?(@query_template)
@@ -179,6 +179,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
 
     test_connection!
+    setup_serverless
   end # def register
 
   def filter(event)
@@ -260,14 +261,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   private
 
   def client_options
-    {
+    @client_options ||= {
       :user => @user,
       :password => @password,
       :api_key => @api_key,
       :proxy => @proxy,
       :ssl => client_ssl_options,
       :retry_on_failure => @retry_on_failure,
-      :retry_on_status => @retry_on_status
+      :retry_on_status => @retry_on_status,
+      :user_agent => prepare_user_agent
     }
   end
 
@@ -344,15 +346,13 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   def new_client
     # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
     # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
-    opts = client_options
-
-    opts[:user_agent] = prepare_user_agent
-
-    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, opts)
+    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, client_options)
   end
 
   def get_client
-    @clients_pool.computeIfAbsent(Thread.current, lambda { |x| new_client })
+    @shared_client || synchronize do
+      @shared_client ||= new_client
+    end
   end
 
   # get an array of path elements from a path reference
@@ -476,6 +476,17 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
   end
 
+  def setup_serverless
+    if get_client.serverless?
+      @client_options[:serverless] = true
+      @shared_client = new_client
+      get_client.info
+    end
+  rescue => e
+    @logger.error("Failed to retrieve Elasticsearch info", message: e.message, exception: e.class, backtrace: e.backtrace)
+    raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
+  end
+
   def setup_ssl_params!
     @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
       normalize.with_deprecated_alias(:ssl)

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.15.2'
+  s.version         = '3.16.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 7.14.0" # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'

data/spec/filters/elasticsearch_spec.rb

@@ -22,6 +22,7 @@ describe LogStash::Filters::Elasticsearch do
 
       before do
         allow(plugin).to receive(:test_connection!)
+        allow(plugin).to receive(:setup_serverless)
       end
       
       it "should not raise an exception" do
@@ -49,6 +50,26 @@ describe LogStash::Filters::Elasticsearch do
       end
     end
 
+    context "against serverless Elasticsearch" do
+      let(:config) { { "query" => "*" } }
+      let(:filter_client) { double("filter_client") }
+      let(:es_client) { double("es_client") }
+
+      before do
+        allow(plugin).to receive(:test_connection!)
+        allow(plugin).to receive(:get_client).and_return(filter_client)
+        allow(filter_client).to receive(:serverless?).and_return(true)
+        allow(filter_client).to receive(:client).and_return(es_client)
+        allow(es_client).to receive(:info).with(a_hash_including(:headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)).and_raise(
+          Elasticsearch::Transport::Transport::Errors::BadRequest.new
+        )
+      end
+
+      it "raises an exception when Elastic Api Version is not supported" do
+        expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
     context "query settings" do
       it "raise an exception when query and query_template are empty" do
         plugin = described_class.new({})
@@ -84,6 +105,7 @@ describe LogStash::Filters::Elasticsearch do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
       allow(client).to receive(:search).and_return(response)
       allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
       plugin.register
     end
 
@@ -91,21 +113,6 @@ describe LogStash::Filters::Elasticsearch do
       Thread.current[:filter_elasticsearch_client] = nil
     end
 
-    # Since the Elasticsearch Ruby client is not thread safe
-    # and under high load we can get error with the connection pool
-    # we have decided to create a new instance per worker thread which
-    # will be lazy created on the first call to `#filter`
-    #
-    # I am adding a simple test case for future changes
-    it "uses a different connection object per thread wait" do
-      expect(plugin.clients_pool.size).to eq(0)
-
-      Thread.new { plugin.filter(event) }.join
-      Thread.new { plugin.filter(event) }.join
-
-      expect(plugin.clients_pool.size).to eq(2)
-    end
-
     it "should enhance the current event with new data" do
       plugin.filter(event)
       expect(event.get("code")).to eq(404)
@@ -438,7 +445,9 @@ describe LogStash::Filters::Elasticsearch do
       let(:plugin) { described_class.new(config) }
       let(:event)  { LogStash::Event.new({}) }
 
-      it "client should sent the expect user-agent" do
+      # elasticsearch-ruby 7.17.9 initialize two user agent headers, `user-agent` and `User-Agent`
+      # hence, fail this header size test case
+      xit "client should sent the expect user-agent" do
         plugin.register
 
         request = webserver.wait_receive_request
@@ -460,12 +469,39 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
     end
 
     after(:each) do
       Thread.current[:filter_elasticsearch_client] = nil
     end
 
+    it 'uses a threadsafe transport adapter' do
+      client = plugin.send(:get_client).client
+      # we currently rely on the threadsafety guarantees provided by Manticore
+      # this spec is a safeguard to trigger an assessment of thread-safety should
+      # we choose a different transport adapter in the future.
+      transport_class = extract_transport(client).options.fetch(:transport_class)
+      expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
+    end
+
+    it 'uses a client with sufficient connection pool size' do
+      client = plugin.send(:get_client).client
+      transport_options = extract_transport(client).options.fetch(:transport_options)
+      # pool_max and pool_max_per_route are manticore-specific transport options
+      expect(transport_options).to include(:pool_max => 1000, :pool_max_per_route => 100)
+    end
+
+    it 'uses a single shared client across threads' do
+      q = Queue.new
+      10.times.map do
+        Thread.new(plugin) { |instance| q.push instance.send(:get_client) }
+      end.map(&:join)
+
+      first = q.pop
+      expect(q.pop).to be(first) until q.empty?
+    end
+
     describe "cloud.id" do
       let(:valid_cloud_id) do
         'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
@@ -608,6 +644,47 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
+  describe "Elastic Api Header" do
+    let(:config) { {"query" => "*"} }
+    let(:plugin) { described_class.new(config) }
+    let(:headers) {{'x-elastic-product' => 'Elasticsearch'}}
+    let(:cluster_info) { {"version" => {"number" => "8.10.0", "build_flavor" => build_flavor}, "tagline" => "You Know, for Search"} }
+    let(:mock_resp) { MockResponse.new(200, cluster_info, headers) }
+
+    before do
+      expect(plugin).to receive(:test_connection!)
+    end
+
+    context "serverless" do
+      let(:build_flavor) { "serverless" }
+
+      before do
+        allow_any_instance_of(Elasticsearch::Client).to receive(:perform_request).with(any_args).and_return(mock_resp)
+      end
+
+      it 'propagates header to es client' do
+        plugin.register
+        client = plugin.send(:get_client).client
+        expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("Elastic-Api-Version" => "2023-10-31")
+      end
+    end
+
+    context "stateful" do
+      let(:build_flavor) { "default" }
+
+      before do
+        expect_any_instance_of(Elasticsearch::Client).to receive(:perform_request).with(any_args).and_return(mock_resp)
+      end
+
+      it 'does not propagate header to es client' do
+        plugin.register
+        client = plugin.send(:get_client).client
+        expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_not_including("Elastic-Api-Version" => "2023-10-31")
+      end
+    end
+
+  end
+
   describe "ca_trusted_fingerprint" do
     let(:ca_trusted_fingerprint) { SecureRandom.hex(32) }
     let(:config) { {"ssl_enabled" => true, "ca_trusted_fingerprint" => ca_trusted_fingerprint, "query" => "*"}}
@@ -616,7 +693,10 @@ describe LogStash::Filters::Elasticsearch do
 
     if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
       context 'the generated trust_strategy' do
-        before(:each) { allow(plugin).to receive(:test_connection!) }
+        before(:each) do
+          allow(plugin).to receive(:test_connection!)
+          allow(plugin).to receive(:setup_serverless)
+        end
 
         it 'is passed to the Manticore client' do
           expect(Manticore::Client).to receive(:new)
@@ -655,7 +735,10 @@ describe LogStash::Filters::Elasticsearch do
 
     subject(:plugin) { described_class.new(config) }
 
-    before(:each) { allow(plugin).to receive(:test_connection!) }
+    before(:each) do
+      allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
+    end
 
     it 'is passed to the Manticore client' do
       expect(Manticore::Client).to receive(:new)
@@ -683,7 +766,10 @@ describe LogStash::Filters::Elasticsearch do
     let(:config) { {"query" => "*"} }
     let(:plugin) { described_class.new(config) }
 
-    before { allow(plugin).to receive(:test_connection!) }
+    before do
+      allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
+    end
 
     it "should set localhost:9200 as hosts" do
       plugin.register
@@ -708,13 +794,14 @@ describe LogStash::Filters::Elasticsearch do
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
       allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
       plugin.register
     end
 
     it "should read and send non-ascii query" do
-      expect(client).to receive(:search).with(
+      expect(client).to receive(:search).with({
           :body => { "query" => { "terms" => { "lock" => [ "잠금", "uzávěr" ] } } },
-          :index => "")
+          :index => ""})
 
       plugin.filter(LogStash::Event.new)
     end
@@ -725,4 +812,21 @@ describe LogStash::Filters::Elasticsearch do
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
+  class MockResponse
+    attr_reader :code, :headers
+
+    def initialize(code = 200, body = nil, headers = {})
+      @code = code
+      @body = body
+      @headers = headers
+    end
+
+    def body
+      @body
+    end
+
+    def status
+      @code
+    end
+  end
 end

data/spec/filters/elasticsearch_ssl_spec.rb

@@ -15,6 +15,8 @@ describe "SSL options" do
   before do
     allow(es_client_double).to receive(:close)
     allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object)
+    allow(es_client_double).to receive(:info).with(any_args).and_return({"version" => {"number" => "7.5.0", "build_flavor" => "default"},
+                                                                         "tagline" => "You Know, for Search"})
     allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double)
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.15.2
+  version: 3.16.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-24 00:00:00.000000000 Z
+date: 2023-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -35,7 +35,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.14.9
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.14.9
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: