logstash-filter-elasticsearch 3.17.1 → 3.19.0

data/spec/filters/elasticsearch_spec.rb

@@ -61,7 +61,7 @@ describe LogStash::Filters::Elasticsearch do
         allow(filter_client).to receive(:serverless?).and_return(true)
         allow(filter_client).to receive(:client).and_return(es_client)
 
-        if elastic_ruby_v8_client_available?
+        if defined?(Elastic::Transport)
           allow(es_client).to receive(:info)
                                 .with(a_hash_including(
                                         :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER))
@@ -93,306 +93,6 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
-  describe "data fetch" do
-    let(:config) do
-      {
-        "hosts" => ["localhost:9200"],
-        "query" => "response: 404",
-        "fields" => { "response" => "code" },
-        "docinfo_fields" => { "_index" => "es_index" },
-        "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
-      }
-    end
-
-    let(:response) do
-      LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
-    end
-
-    let(:client) { double(:client) }
-
-    before(:each) do
-      allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
-      if elastic_ruby_v8_client_available?
-        allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
-      else
-        allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
-      end
-      allow(client).to receive(:search).and_return(response)
-      allow(plugin).to receive(:test_connection!)
-      allow(plugin).to receive(:setup_serverless)
-      plugin.register
-    end
-
-    after(:each) do
-      Thread.current[:filter_elasticsearch_client] = nil
-    end
-
-    it "should enhance the current event with new data" do
-      plugin.filter(event)
-      expect(event.get("code")).to eq(404)
-      expect(event.get("es_index")).to eq("logstash-2014.08.26")
-      expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
-    end
-
-    it "should receive all necessary params to perform the search" do
-      expect(client).to receive(:search).with({:q=>"response: 404", :size=>1, :index=>"", :sort=>"@timestamp:desc"})
-      plugin.filter(event)
-    end
-
-    context "when asking to hit specific index" do
-
-      let(:config) do
-        {
-          "index" => "foo*",
-          "hosts" => ["localhost:9200"],
-          "query" => "response: 404",
-          "fields" => { "response" => "code" }
-        }
-      end
-
-      it "should receive all necessary params to perform the search" do
-        expect(client).to receive(:search).with({:q=>"response: 404", :size=>1, :index=>"foo*", :sort=>"@timestamp:desc"})
-        plugin.filter(event)
-      end
-    end
-
-    context "when asking for more than one result" do
-
-      let(:config) do
-        {
-          "hosts" => ["localhost:9200"],
-          "query" => "response: 404",
-          "fields" => { "response" => "code" },
-          "result_size" => 10
-        }
-      end
-
-      let(:response) do
-        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
-      end
-
-      it "should enhance the current event with new data" do
-        plugin.filter(event)
-        expect(event.get("code")).to eq([404]*10)
-      end
-    end
-
-    context 'when Elasticsearch 7.x gives us a totals object instead of an integer' do
-      let(:config) do
-        {
-            "hosts" => ["localhost:9200"],
-            "query" => "response: 404",
-            "fields" => { "response" => "code" },
-            "result_size" => 10
-        }
-      end
-
-      let(:response) do
-        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "elasticsearch_7.x_hits_total_as_object.json")))
-      end
-
-      it "should enhance the current event with new data" do
-        plugin.filter(event)
-        expect(event.get("[@metadata][total_hits]")).to eq(13476)
-      end
-    end
-
-    context "if something wrong happen during connection" do
-
-      before(:each) do
-        allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
-        allow(client).to receive(:search).and_raise("connection exception")
-        plugin.register
-      end
-
-      it "tag the event as something happened, but still deliver it" do
-        expect(plugin.logger).to receive(:warn)
-        plugin.filter(event)
-        expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
-      end
-    end
-
-    # Tagging test for positive results
-    context "Tagging should occur if query returns results" do
-      let(:config) do
-        {
-          "index" => "foo*",
-          "hosts" => ["localhost:9200"],
-          "query" => "response: 404",
-          "add_tag" => ["tagged"]
-        }
-      end
-
-      let(:response) do
-        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
-      end
-
-      it "should tag the current event if results returned" do
-        plugin.filter(event)
-        expect(event.to_hash["tags"]).to include("tagged")
-      end
-    end
-
-    context "an aggregation search with size 0 that matches" do
-      let(:config) do
-        {
-          "index" => "foo*",
-          "hosts" => ["localhost:9200"],
-          "query" => "response: 404",
-          "add_tag" => ["tagged"],
-          "result_size" => 0,
-          "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
-        }
-      end
-
-      let(:response) do
-        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_size0_agg.json")))
-      end
-
-      it "should tag the current event" do
-        plugin.filter(event)
-        expect(event.get("tags")).to include("tagged")
-        expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
-      end
-    end
-
-    # Tagging test for negative results
-    context "Tagging should not occur if query has no results" do
-      let(:config) do
-        {
-          "index" => "foo*",
-          "hosts" => ["localhost:9200"],
-          "query" => "response: 404",
-          "add_tag" => ["tagged"]
-        }
-      end
-
-      let(:response) do
-        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
-      end
-
-      it "should not tag the current event" do
-        plugin.filter(event)
-        expect(event.to_hash["tags"]).to_not include("tagged")
-      end
-    end
-    context "testing a simple query template" do
-      let(:config) do
-        {
-          "hosts" => ["localhost:9200"],
-          "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
-          "fields" => { "response" => "code" },
-          "result_size" => 1
-        }
-      end
-
-      let(:response) do
-        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
-      end
-
-      it "should enhance the current event with new data" do
-        plugin.filter(event)
-        expect(event.get("code")).to eq(404)
-      end
-
-    end
-
-    context "testing a simple index substitution" do
-      let(:event) {
-        LogStash::Event.new(
-            {
-                "subst_field" => "subst_value"
-            }
-        )
-      }
-      let(:config) do
-        {
-            "index" => "foo_%{subst_field}*",
-            "hosts" => ["localhost:9200"],
-            "query" => "response: 404",
-            "fields" => { "response" => "code" }
-        }
-      end
-
-      it "should receive substituted index name" do
-        expect(client).to receive(:search).with({:q => "response: 404", :size => 1, :index => "foo_subst_value*", :sort => "@timestamp:desc"})
-        plugin.filter(event)
-      end
-    end
-
-    context "if query result errored but no exception is thrown" do
-      let(:response) do
-        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
-      end
-
-      before(:each) do
-        allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
-        allow(client).to receive(:search).and_return(response)
-        plugin.register
-      end
-
-      it "tag the event as something happened, but still deliver it" do
-        expect(plugin.logger).to receive(:warn)
-        plugin.filter(event)
-        expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
-      end
-    end
-
-    context 'with client-level retries' do
-      let(:config) do
-        super().merge(
-          "retry_on_failure" => 3,
-          "retry_on_status" => [500]
-        )
-      end
-    end
-
-    context "with custom headers" do
-      let(:config) do
-        {
-          "query" => "*",
-          "custom_headers" => { "Custom-Header-1" => "Custom Value 1", "Custom-Header-2" => "Custom Value 2" }
-        }
-      end
-
-      let(:plugin) { LogStash::Filters::Elasticsearch.new(config) }
-      let(:client_double) { double("client") }
-      let(:transport_double) { double("transport", options: { transport_options: { headers: config["custom_headers"] } }) }
-
-      before do
-        allow(plugin).to receive(:get_client).and_return(client_double)
-        if elastic_ruby_v8_client_available?
-          allow(client_double).to receive(:es_transport_client_type).and_return('elastic_transport')
-        else
-          allow(client_double).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
-        end
-        allow(client_double).to receive(:client).and_return(transport_double)
-      end
-
-      it "sets custom headers" do
-        plugin.register
-        client = plugin.send(:get_client).client
-        expect(client.options[:transport_options][:headers]).to match(hash_including(config["custom_headers"]))
-      end
-    end
-
-    context "if query is on nested field" do
-      let(:config) do
-        {
-            "hosts" => ["localhost:9200"],
-            "query" => "response: 404",
-            "fields" => [ ["[geoip][ip]", "ip_address"] ]
-        }
-      end
-
-      it "should enhance the current event with new data" do
-        plugin.filter(event)
-        expect(event.get("ip_address")).to eq("66.249.73.185")
-      end
-
-    end
-  end
-
   class StoppableServer
 
     attr_reader :port
@@ -525,7 +225,7 @@ describe LogStash::Filters::Elasticsearch do
       # this spec is a safeguard to trigger an assessment of thread-safety should
       # we choose a different transport adapter in the future.
       transport_class = extract_transport(client).options.fetch(:transport_class)
-      if elastic_ruby_v8_client_available?
+      if defined?(Elastic::Transport)
         allow(client).to receive(:es_transport_client_type).and_return("elastic_transport")
         expect(transport_class).to equal ::Elastic::Transport::Transport::HTTP::Manticore
       else
@@ -845,7 +545,7 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
-      if elastic_ruby_v8_client_available?
+      if defined?(Elastic::Transport)
         allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
       else
         allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
@@ -864,19 +564,149 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
+  describe "ES|QL" do
+
+    describe "compatibility" do
+      let(:config) {{ "hosts" => ["localhost:9200"], "query_type" => "esql", "query" => "FROM my-index" }}
+
+      context "when LS doesn't support ES|QL" do
+        let(:ls_version) { LogStash::Filters::Elasticsearch::LS_ESQL_SUPPORT_VERSION }
+        before(:each) do
+          stub_const("LOGSTASH_VERSION", "8.17.0")
+        end
+
+        it "raises a runtime error" do
+          expect { plugin.send(:validate_ls_version_for_esql_support!) }
+            .to raise_error(RuntimeError, /Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{ls_version}/)
+        end
+      end
+
+      context "when ES doesn't support ES|QL" do
+        let(:es_version) { LogStash::Filters::Elasticsearch::ES_ESQL_SUPPORT_VERSION }
+        let(:client) { double(:client) }
+
+        it "raises a runtime error" do
+          allow(plugin).to receive(:get_client).twice.and_return(client)
+          allow(client).to receive(:es_version).and_return("8.8.0")
+
+          expect { plugin.send(:validate_es_for_esql_support!) }
+            .to raise_error(RuntimeError, /Connected Elasticsearch 8.8.0 version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{es_version} version./)
+        end
+      end
+    end
+
+    context "when non-ES|QL params applied" do
+      let(:config) do
+        {
+          "hosts" => ["localhost:9200"],
+          "query_type" => "esql",
+          "query" => "FROM my-index",
+          "index" => "some-index",
+          "docinfo_fields" => { "_index" => "es_index" },
+          "sort" => "@timestamp:desc",
+          "enable_sort" => true,
+          "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
+        }
+      end
+      it "raises a config error" do
+        invalid_params_with_esql = %w(index docinfo_fields sort enable_sort aggregation_fields)
+        error_text = /Configured #{invalid_params_with_esql} params cannot be used with ES|QL query/i
+        expect { plugin.register }.to raise_error LogStash::ConfigurationError, error_text
+      end
+    end
+
+    describe "#query placeholder" do
+      let(:config) do
+        {
+          "hosts" => ["localhost:9200"],
+          "query_type" => "esql"
+        }
+      end
+
+      context "when query placeholder doesn't exist in the query" do
+        let(:config) {
+          super()
+            .merge(
+              {
+                "query" => "FROM my-index",
+                "query_params" => { "a" => "b" },
+              })
+        }
+
+        it "doesn't complain since not used" do
+          expect { plugin.send(:validate_esql_query_and_params!) }.not_to raise_error
+        end
+      end
+
+      context "when illegal placeholders appear" do
+        let(:config) {
+          super()
+            .merge(
+              {
+                "query" => "FROM my-index | WHERE type = ?type",
+                "query_params" => { "1abcd_efg1" => "1", "$abcd_efg1" => 2, "type" => 3 },
+              })
+        }
+        it "raises a config error" do
+          message = 'Illegal ["1abcd_efg1", "$abcd_efg1"] placeholder names in `query_params`. A valid parameter name starts with a letter and contains letters, digits and underscores only;'
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, message
+        end
+      end
+
+      context "when query placeholders and `query_params` do not match" do
+        let(:config) {
+          super()
+            .merge(
+              {
+                "query" => "FROM my-index | WHERE type = ?type",
+                "query_params" => {"b" => "c"},
+              })
+        }
+        it "raises a config error" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Placeholder type not found in query/
+        end
+      end
+
+      context "when `query_params` is an Array contains {key => val} entries" do
+        let(:config) {
+          super()
+            .merge(
+              {
+                "query" => "FROM my-index",
+                "query_params" => [{ "a" => "b" }, { "c" => "[b]" }, { "e" => 1 }, { "f" => "[g]" }],
+              })
+        }
+
+        it "doesn't complain since not used" do
+          expect { plugin.send(:validate_esql_query_and_params!) }.not_to raise_error
+          expect(plugin.query_params).to eq({ "a" => "b", "c" => "[b]", "e" => 1, "f" => "[g]" })
+        end
+      end
+
+      context "when `query_params` is a Hash" do
+        let(:config) {
+          super()
+            .merge(
+              {
+                "query" => "FROM my-index",
+                "query_params" => { "a" => "b", "c" => "[b]", "e" => 1, "f" => "[g]" },
+              })
+        }
+
+        it "doesn't complain since not used" do
+          expect { plugin.send(:validate_esql_query_and_params!) }.not_to raise_error
+          expect(plugin.query_params).to eq({ "a" => "b", "c" => "[b]", "e" => 1, "f" => "[g]" })
+        end
+      end
+    end if LOGSTASH_VERSION >= '8.17.4'
+  end
+
   def extract_transport(client)
     # on 7x: client.transport.transport
     # on >=8.x: client.transport
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
-  def elastic_ruby_v8_client_available?
-    Elasticsearch::Transport
-    false
-  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
-    true
-  end
-
   class MockResponse
     attr_reader :code, :headers
 

data/spec/filters/integration/elasticsearch_esql_spec.rb

@@ -0,0 +1,167 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/elasticsearch"
+require "elasticsearch"
+require_relative "../../../spec/es_helper"
+
+describe LogStash::Filters::Elasticsearch, integration: true do
+
+  ELASTIC_SECURITY_ENABLED = ENV['ELASTIC_SECURITY_ENABLED'].eql? 'true'
+  SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
+  ES_HOSTS = ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"]
+  CA_PATH = File.expand_path('../fixtures/test_certs/ca.crt', File.dirname(__FILE__))
+
+  let(:plugin) { described_class.new(config) }
+  let(:es_index) { "es-filter-plugin-esql-integration-#{rand(1000)}" }
+  let(:test_documents) do
+    [
+      { "message" => "test message 1", "type" => "a", "count" => 1 },
+      { "message" => "test message 2", "type" => "a", "count" => 2 },
+      { "message" => "test message 3", "type" => "b", "count" => 3 },
+      { "message" => "test message 4", "type" => "b", "count" => 4 },
+      { "message" => "test message 5", "type" => "c", "count" => 5 },
+      { "message" => "odd test message", "type" => "t" }
+    ]
+  end
+
+  let(:base_config) do
+    {
+      "query_type" => "esql",
+      "hosts" => ES_HOSTS,
+      "ssl_enabled" => SECURE_INTEGRATION
+    }
+  end
+
+  let(:credentials) do
+    if SECURE_INTEGRATION
+      { 'user' => 'tests', 'password' => 'Tests123' }
+    else
+      { 'user' => 'elastic', 'password' => ENV['ELASTIC_PASSWORD'] }
+    end
+  end
+
+  let(:config) do
+    config = ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config
+    config = { 'ssl_certificate_authorities' => CA_PATH }.merge(config) if SECURE_INTEGRATION
+    config
+  end
+
+  let(:event) { LogStash::Event.new({}) }
+
+  def es_client
+    @es_client ||= begin
+      user = SECURE_INTEGRATION ? 'tests' : 'elastic'
+      password = SECURE_INTEGRATION ? 'Tests123' : ENV['ELASTIC_PASSWORD']
+
+      es_client_config = { hosts: ES_HOSTS }
+      es_client_config = es_client_config.merge({ user: user, password: password }) if ELASTIC_SECURITY_ENABLED || SECURE_INTEGRATION
+      es_client_config = es_client_config.merge({ transport_options: { ssl: { ca_path: CA_PATH, verify: false }}}) if SECURE_INTEGRATION
+
+      Elasticsearch::Client.new(es_client_config)
+    end
+  end
+
+  before(:all) do
+    is_ls_with_esql_supported_client = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(LogStash::Filters::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
+    # Skip tests if an ES version doesn't support ES|QL
+    skip "LS version does not have ES client which supports ES|QL" unless is_ls_with_esql_supported_client
+
+    es_version_info = es_client.info["version"]
+    es_gem_version = Gem::Version.create(es_version_info["number"])
+    skip "ES version does not support ES|QL" if es_gem_version.nil? || es_gem_version < Gem::Version.create(LogStash::Filters::Elasticsearch::ES_ESQL_SUPPORT_VERSION)
+  end
+
+  before(:each) do
+    # Create index with test documents
+    es_client.indices.create(index: es_index, body: {}) unless es_client.indices.exists?(index: es_index)
+
+    test_documents.each do |doc|
+      es_client.index(index: es_index, body: doc, refresh: true)
+    end
+  end
+
+  after(:each) do
+    es_client.indices.delete(index: es_index) if es_client.indices.exists?(index: es_index)
+  end
+
+  describe "run ES|QL queries" do
+
+    before do
+      stub_const("LOGSTASH_VERSION", LogStash::Filters::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
+    end
+
+    before(:each) do
+      plugin.register
+    end
+
+    shared_examples "ESQL query execution" do |expected_count, fields|
+      it "processes the event" do
+        plugin.filter(event)
+        expect(event.get("[@metadata][total_values]")).to eq(expected_count)
+        fields&.each do | field |
+          expect(event.get(field)).not_to be(nil)
+        end
+      end
+    end
+
+    describe "with simple FROM query with LIMIT" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | LIMIT 99")
+      end
+
+      include_examples "ESQL query execution", 6
+    end
+
+    describe "with simple FROM and WHERE query combinations" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | WHERE type==\"b\" | LIMIT 99")
+      end
+
+      include_examples "ESQL query execution", 2
+    end
+
+    describe "with query params" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | WHERE type==?type", "query_params" => { "type" => "b" })
+      end
+
+      include_examples "ESQL query execution", 2
+    end
+
+    describe "when invalid query used" do
+      let(:config) do
+        super().merge("query" => "FROM undefined index | LIMIT 1")
+      end
+
+      it "tags on failure" do
+        plugin.filter(event)
+        expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
+      end
+    end
+
+    describe "when field enrichment requested" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | WHERE type==\"b\" | LIMIT 99")
+      end
+
+      include_examples "ESQL query execution", 2,  %w[message count]
+    end
+
+    describe "when non-exist field value appear" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index}", "target" => "target_field")
+      end
+
+      it "processes the event" do
+        plugin.filter(event)
+        expect(event.get("[@metadata][total_values]")).to eq(6)
+        expect(event.get("target_field").size).to eq(6)
+        values = event.get("target_field")
+        counts = values.count { |entry| entry.key?("count") }
+        messages = values.count { |entry| entry.key?("message") }
+        expect(counts).to eq(5)
+        expect(messages).to eq(6)
+      end
+    end
+  end
+end

data/spec/filters/integration/elasticsearch_spec.rb

@@ -84,7 +84,9 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
     end
 
     it "fails to register plugin" do
-      expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+      expect { plugin.register }.to raise_error elastic_ruby_v8_client_available? ?
+                                                  Elastic::Transport::Transport::Errors::Unauthorized :
+                                                  Elasticsearch::Transport::Transport::Errors::Unauthorized
     end
 
   end if ELASTIC_SECURITY_ENABLED
@@ -150,5 +152,10 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
       end
     end
   end
-
+  def elastic_ruby_v8_client_available?
+    Elasticsearch::Transport
+    false
+  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
+    true
+  end
 end