logstash-filter-elasticsearch 3.15.1 → 3.15.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a80d8c888e4c85fad00861c48191031b12827b75d4f7adf2f82d3be8023ef4c
-  data.tar.gz: ecf9e2cf9c3d020bc5ece06d2b430bef230f60f2505847a534556fd95d72094f
+  metadata.gz: 7e85d9b1122e7e095666cdc3bfa1dd5fbd3ef62ce5cf15d667ffa92932e5748e
+  data.tar.gz: cbcf76ec5d048bb18f467520f4a646a08e5ea39a8d44ace7a3322e7fea0eee06
 SHA512:
-  metadata.gz: c02acdc7b189b40b9d53f32374c982a0239a2be8a5e6d10f4ec62ee99771822e3a8b0345e432f5fd488eaed02e8f9207b82b4226e29f310bbbfc2f786eafd6bc
-  data.tar.gz: 25cfab8d4069d7e30c5f1dd113857c24ce8c43ef7ed566b66230c64ead218398e28f504b7c0f5d864a8a6174cbf817e1f76faa22d996f15cfa0e298d3ed6e334
+  metadata.gz: 6bc45e8790eede5047013506fe8b9f8ba97ad08365f6ab99d5fd7ca120e051f916301054e23932d068823ba8f0904757de9bc0bb255d13b1967b339c600e4011
+  data.tar.gz: '07842227fb3c356f08012193925fe1b5ae055ead761600c029e0ea4a788b97744113e35ebf97c05e223270e14017bf1d10b66a0ca038a810ca5054c1c2ff8389'

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.15.3
+  - Fixes a memory leak that occurs when a pipeline containing this filter terminates, which could become significant if the pipeline is cycled repeatedly [#173](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/173)
+
+## 3.15.2
+  - Added checking for `query` and `query_template`. [#171](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/171)
+  
 ## 3.15.1
   - Fixes a regression introduced in 3.15.0 which could prevent a connection from being established to Elasticsearch in some SSL configurations
 

data/docs/index.asciidoc

@@ -320,6 +320,7 @@ environment variables e.g. `proxy => '${LS_PROXY:}'`.
 Elasticsearch query string. More information is available in the
 {ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query
 string documentation].
+Use either `query` or `query_template`.
 
 
 [id="plugins-{type}s-{plugin}-query_template"]
@@ -330,6 +331,7 @@ string documentation].
 
 File path to elasticsearch query in DSL format. More information is available in
 the {ref}/query-dsl.html[Elasticsearch query documentation].
+Use either `query` or `query_template`.
 
 [id="plugins-{type}s-{plugin}-result_size"]
 ===== `result_size` 

data/lib/logstash/filters/elasticsearch/client.rb

@@ -22,6 +22,9 @@ module LogStash
         transport_options[:headers].merge!(setup_api_key(api_key))
         transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
 
+        transport_options[:pool_max] = 1000
+        transport_options[:pool_max_per_route] = 100
+
         logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
         transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')
 

data/lib/logstash/filters/elasticsearch.rb

@@ -4,6 +4,7 @@ require "logstash/namespace"
 require "logstash/json"
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "logstash/plugin_mixins/normalize_config_support"
+require "monitor"
 
 require_relative "elasticsearch/client"
 require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
@@ -139,7 +140,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   include LogStash::PluginMixins::NormalizeConfigSupport
 
-  attr_reader :clients_pool
+  include MonitorMixin
+  attr_reader :shared_client
 
   ##
   # @override to handle proxy => '' as if none was set
@@ -159,8 +161,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def register
-    @clients_pool = java.util.concurrent.ConcurrentHashMap.new
-
     #Load query if it exists
     if @query_template
       if File.zero?(@query_template)
@@ -170,6 +170,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       @query_dsl = file.read
     end
 
+    validate_query_settings
     fill_hosts_from_cloud_id
     setup_ssl_params!
     validate_authentication
@@ -351,7 +352,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def get_client
-    @clients_pool.computeIfAbsent(Thread.current, lambda { |x| new_client })
+    @shared_client || synchronize do
+      @shared_client ||= new_client
+    end
   end
 
   # get an array of path elements from a path reference
@@ -391,6 +394,16 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     hosts.is_a?(Array) && hosts.size == 1 && !original_params.key?('hosts')
   end
 
+  def validate_query_settings
+    unless @query || @query_template
+      raise LogStash::ConfigurationError, "Both `query` and `query_template` are empty. Require either `query` or `query_template`."
+    end
+
+    if @query && @query_template
+      raise LogStash::ConfigurationError, "Both `query` and `query_template` are set. Use either `query` or `query_template`."
+    end
+  end
+
   def validate_authentication
     authn_options = 0
     authn_options += 1 if @cloud_auth

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.15.1'
+  s.version         = '3.15.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -15,9 +15,11 @@ describe LogStash::Filters::Elasticsearch do
 
   context "registration" do
 
-    let(:plugin) { LogStash::Plugin.lookup("filter", "elasticsearch").new({}) }
+    let(:plugin) { LogStash::Plugin.lookup("filter", "elasticsearch").new(config) }
 
     context "against authentic Elasticsearch" do
+      let(:config) { { "query" => "*" } }
+
       before do
         allow(plugin).to receive(:test_connection!)
       end
@@ -28,6 +30,7 @@ describe LogStash::Filters::Elasticsearch do
     end
 
     context "against not authentic Elasticsearch" do
+      let(:config) { { "query" => "*" } }
       let(:failing_client) do
         client = double("client")
         allow(client).to receive(:ping).and_raise Elasticsearch::UnsupportedProductError
@@ -45,6 +48,19 @@ describe LogStash::Filters::Elasticsearch do
         expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
       end
     end
+
+    context "query settings" do
+      it "raise an exception when query and query_template are empty" do
+        plugin = described_class.new({})
+        expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+      end
+
+      it "raise an exception when query and query_template are set" do
+        config = { "query" => "*", "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template_unicode.json") }
+        plugin = described_class.new(config)
+        expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+      end
+    end
   end
 
   describe "data fetch" do
@@ -75,21 +91,6 @@ describe LogStash::Filters::Elasticsearch do
       Thread.current[:filter_elasticsearch_client] = nil
     end
 
-    # Since the Elasticsearch Ruby client is not thread safe
-    # and under high load we can get error with the connection pool
-    # we have decided to create a new instance per worker thread which
-    # will be lazy created on the first call to `#filter`
-    #
-    # I am adding a simple test case for future changes
-    it "uses a different connection object per thread wait" do
-      expect(plugin.clients_pool.size).to eq(0)
-
-      Thread.new { plugin.filter(event) }.join
-      Thread.new { plugin.filter(event) }.join
-
-      expect(plugin.clients_pool.size).to eq(2)
-    end
-
     it "should enhance the current event with new data" do
       plugin.filter(event)
       expect(event.get("code")).to eq(404)
@@ -450,6 +451,32 @@ describe LogStash::Filters::Elasticsearch do
       Thread.current[:filter_elasticsearch_client] = nil
     end
 
+    it 'uses a threadsafe transport adapter' do
+      client = plugin.send(:get_client).client
+      # we currently rely on the threadsafety guarantees provided by Manticore
+      # this spec is a safeguard to trigger an assessment of thread-safety should
+      # we choose a different transport adapter in the future.
+      transport_class = extract_transport(client).options.fetch(:transport_class)
+      expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
+    end
+
+    it 'uses a client with sufficient connection pool size' do
+      client = plugin.send(:get_client).client
+      transport_options = extract_transport(client).options.fetch(:transport_options)
+      # pool_max and pool_max_per_route are manticore-specific transport options
+      expect(transport_options).to include(:pool_max => 1000, :pool_max_per_route => 100)
+    end
+
+    it 'uses a single shared client across threads' do
+      q = Queue.new
+      10.times.map do
+        Thread.new(plugin) { |instance| q.push instance.send(:get_client) }
+      end.map(&:join)
+
+      first = q.pop
+      expect(q.pop).to be(first) until q.empty?
+    end
+
     describe "cloud.id" do
       let(:valid_cloud_id) do
         'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
@@ -594,7 +621,7 @@ describe LogStash::Filters::Elasticsearch do
 
   describe "ca_trusted_fingerprint" do
     let(:ca_trusted_fingerprint) { SecureRandom.hex(32) }
-    let(:config) { {"ssl_enabled" => true, "ca_trusted_fingerprint" => ca_trusted_fingerprint}}
+    let(:config) { {"ssl_enabled" => true, "ca_trusted_fingerprint" => ca_trusted_fingerprint, "query" => "*"}}
 
     subject(:plugin) { described_class.new(config) }
 
@@ -633,6 +660,7 @@ describe LogStash::Filters::Elasticsearch do
         'hosts' => 'https://localhost:9200',
         'ssl_keystore_path' => keystore_path,
         'ssl_keystore_password' => keystore_password,
+        'query' => '*'
       }
     end
 
@@ -663,7 +691,7 @@ describe LogStash::Filters::Elasticsearch do
 
   describe "defaults" do
 
-    let(:config) { Hash.new }
+    let(:config) { {"query" => "*"} }
     let(:plugin) { described_class.new(config) }
 
     before { allow(plugin).to receive(:test_connection!) }
@@ -695,9 +723,9 @@ describe LogStash::Filters::Elasticsearch do
     end
 
     it "should read and send non-ascii query" do
-      expect(client).to receive(:search).with(
+      expect(client).to receive(:search).with({
           :body => { "query" => { "terms" => { "lock" => [ "잠금", "uzávěr" ] } } },
-          :index => "")
+          :index => ""})
 
       plugin.filter(LogStash::Event.new)
     end

data/spec/filters/elasticsearch_ssl_spec.rb

@@ -5,7 +5,7 @@ require "logstash/codecs/base"
 describe "SSL options" do
   let(:es_client_double) { double("Elasticsearch::Client #{self.inspect}") }
   let(:hosts) {["localhost"]}
-  let(:settings) { { "ssl_enabled" => true, "hosts" => hosts } }
+  let(:settings) { { "ssl_enabled" => true, "hosts" => hosts, "query" => "*" } }
 
   subject do
     require "logstash/filters/elasticsearch"
@@ -36,7 +36,8 @@ describe "SSL options" do
     context "false and cloud_id resolved host is https" do
       let(:settings) {{
         "ssl_enabled" => false,
-        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=="
+        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==",
+        "query" => "*"
       }}
 
       it "should not infer the ssl_enabled value" do
@@ -82,7 +83,8 @@ describe "SSL options" do
 
     context "and cloud_id resolved host is https" do
       let(:settings) {{
-        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=="
+        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==",
+        "query" => "*"
       }}
 
       it "should infer the ssl_enabled value to false" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.15.1
+  version: 3.15.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-02 00:00:00.000000000 Z
+date: 2023-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement