logstash-input-elasticsearch 4.21.2 → 4.23.0

data/spec/inputs/elasticsearch_spec.rb

@@ -1152,7 +1152,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
     context "when there's an exception" do
       before(:each) do
-        allow(client).to receive(:search).and_raise RuntimeError
+        allow(client).to receive(:search).and_raise RuntimeError.new("test exception")
       end
       it 'produces no events' do
         plugin.run queue
@@ -1297,6 +1297,10 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
     let(:mock_queue) { double('queue', :<< => nil) }
 
+    before(:each) do
+      plugin.send(:setup_cursor_tracker)
+    end
+
     it 'pushes a generated event to the queue' do
       plugin.send(:push_hit, hit, mock_queue)
       expect(mock_queue).to have_received(:<<) do |event|
@@ -1353,4 +1357,129 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
+  describe "#ESQL" do
+    let(:config) do
+      {
+        "query" => "FROM test-index | STATS count() BY field",
+        "query_type" => "esql",
+        "retries" => 3
+      }
+    end
+    let(:es_version) { LogStash::Inputs::Elasticsearch::ES_ESQL_SUPPORT_VERSION }
+    let(:ls_version) { LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION }
+
+    before(:each) do
+      stub_const("LOGSTASH_VERSION", ls_version)
+    end
+
+    describe "#initialize" do
+      it "sets up the ESQL client with correct parameters" do
+        expect(plugin.instance_variable_get(:@query_type)).to eq(config["query_type"])
+        expect(plugin.instance_variable_get(:@query)).to eq(config["query"])
+        expect(plugin.instance_variable_get(:@retries)).to eq(config["retries"])
+      end
+    end
+
+    describe "#register" do
+      before(:each) do
+        Elasticsearch::Client.send(:define_method, :ping) { }
+        allow_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(cluster_info)
+      end
+      it "creates ES|QL executor" do
+        plugin.register
+        expect(plugin.instance_variable_get(:@query_executor)).to be_an_instance_of(LogStash::Inputs::Elasticsearch::Esql)
+      end
+    end
+
+    describe "#validation" do
+
+      describe "LS version" do
+        context "when compatible" do
+
+          it "does not raise an error" do
+            expect { plugin.send(:validate_ls_version_for_esql_support!) }.not_to raise_error
+          end
+        end
+
+        context "when incompatible" do
+          before(:each) do
+            stub_const("LOGSTASH_VERSION", "8.10.0")
+          end
+
+          it "raises a runtime error" do
+            expect { plugin.send(:validate_ls_version_for_esql_support!) }
+              .to raise_error(RuntimeError, /Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{ls_version}/)
+          end
+        end
+      end
+
+      describe "ES version" do
+        before(:each) do
+          allow(plugin).to receive(:es_version).and_return("8.10.5")
+        end
+
+        context "when incompatible" do
+          it "raises a runtime error" do
+            expect { plugin.send(:validate_es_for_esql_support!) }
+              .to raise_error(RuntimeError, /Connected Elasticsearch 8.10.5 version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{es_version} version./)
+          end
+        end
+      end
+
+      context "ES|QL query and DSL params used together" do
+        let(:config) {
+          super().merge({
+            "index" => "my-index",
+            "size" => 1,
+            "slices" => 1,
+            "search_api" => "auto",
+            "docinfo" => true,
+            "docinfo_target" => "[@metadata][docinfo]",
+            "docinfo_fields" => ["_index"],
+            "response_type" => "hits",
+            "tracking_field" => "[@metadata][tracking]"
+          })}
+
+        it "raises a config error" do
+          mixed_fields = %w[index size slices docinfo_fields response_type tracking_field]
+          expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Configured #{mixed_fields} params are not allowed while using ES|QL query/)
+        end
+      end
+
+      describe "ES|QL query"  do
+        context "when query is valid" do
+          it "does not raise an error" do
+            expect { plugin.send(:validate_esql_query!) }.not_to raise_error
+          end
+        end
+
+        context "when query is empty" do
+          let(:config) do
+            {
+              "query" => " "
+            }
+          end
+
+          it "raises a configuration error" do
+            expect { plugin.send(:validate_esql_query!) }
+              .to raise_error(LogStash::ConfigurationError, /`query` cannot be empty/)
+          end
+        end
+
+        context "when query doesn't align with ES syntax" do
+          let(:config) do
+            {
+              "query" => "RANDOM query"
+            }
+          end
+
+          it "raises a configuration error" do
+            source_commands = %w[FROM ROW SHOW]
+            expect { plugin.send(:validate_esql_query!) }
+              .to raise_error(LogStash::ConfigurationError, "`query` needs to start with any of #{source_commands}")
+          end
+        end
+      end
+    end
+  end
 end

data/spec/inputs/integration/elasticsearch_esql_spec.rb

@@ -0,0 +1,150 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/elasticsearch"
+require "elasticsearch"
+require_relative "../../../spec/es_helper"
+
+describe LogStash::Inputs::Elasticsearch, integration: true do
+
+  SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
+  ES_HOSTS = ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"]
+
+  let(:plugin) { described_class.new(config) }
+  let(:es_index) { "logstash-esql-integration-#{rand(1000)}" }
+  let(:test_documents) do
+    [
+      { "message" => "test message 1", "type" => "a", "count" => 1 },
+      { "message" => "test message 2", "type" => "a", "count" => 2 },
+      { "message" => "test message 3", "type" => "b", "count" => 3 },
+      { "message" => "test message 4", "type" => "b", "count" => 4 },
+      { "message" => "test message 5", "type" => "c", "count" => 5 }
+    ]
+  end
+  let(:config) do
+    {
+      "hosts" => ES_HOSTS,
+      "query_type" => "esql"
+    }
+  end
+  let(:es_client) do
+    Elasticsearch::Client.new(hosts: ES_HOSTS)
+  end
+
+  before(:all) do
+    is_ls_with_esql_supported_client = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
+    skip "LS version does not have ES client which supports ES|QL" unless is_ls_with_esql_supported_client
+
+    # Skip tests if ES version doesn't support ES||QL
+    es_client = Elasticsearch::Client.new(hosts: ES_HOSTS) # need to separately create since let isn't allowed in before(:context)
+    es_version_info = es_client.info["version"]
+    es_gem_version = Gem::Version.create(es_version_info["number"])
+    skip "ES version does not support ES|QL" if es_gem_version.nil? || es_gem_version < Gem::Version.create(LogStash::Inputs::Elasticsearch::ES_ESQL_SUPPORT_VERSION)
+  end
+
+  before(:each) do
+    # Create index with test documents
+    es_client.indices.create(index: es_index, body: {}) unless es_client.indices.exists?(index: es_index)
+
+    test_documents.each do |doc|
+      es_client.index(index: es_index, body: doc, refresh: true)
+    end
+  end
+
+  after(:each) do
+    es_client.indices.delete(index: es_index) if es_client.indices.exists?(index: es_index)
+  end
+
+  context "#run ES|QL queries" do
+
+    before do
+      stub_const("LOGSTASH_VERSION", LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
+      allow_any_instance_of(LogStash::Inputs::Elasticsearch).to receive(:exit_plugin?).and_return false, true
+    end
+
+    before(:each) do
+      plugin.register
+    end
+
+    shared_examples "ESQL query execution" do |expected_count|
+      it "correctly retrieves documents" do
+        queue = Queue.new
+        plugin.run(queue)
+
+        event_count = 0
+        expected_count.times do |i|
+          event = queue.pop
+          expect(event).to be_a(LogStash::Event)
+          event_count += 1
+        end
+        expect(event_count).to eq(expected_count)
+      end
+    end
+
+    context "#FROM query" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | SORT count")
+      end
+
+      include_examples "ESQL query execution", 5
+    end
+
+    context "#FROM query and WHERE clause" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | WHERE type == \"a\" | SORT count")
+      end
+
+      include_examples "ESQL query execution", 2
+    end
+
+    context "#STATS aggregation" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | STATS avg(count) BY type")
+      end
+
+      it "retrieves aggregated stats" do
+        queue = Queue.new
+        plugin.run(queue)
+        results = []
+        3.times do
+          event = queue.pop
+          expect(event).to be_a(LogStash::Event)
+          results << event.get("avg(count)")
+        end
+
+        expected_averages = [1.5, 3.5, 5.0]
+        expect(results.sort).to eq(expected_averages)
+      end
+    end
+
+    context "#METADATA" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} METADATA _index, _id, _version | DROP message.keyword, type.keyword | SORT count")
+      end
+
+      it "includes document metadata" do
+        queue = Queue.new
+        plugin.run(queue)
+
+        5.times do
+          event = queue.pop
+          expect(event).to be_a(LogStash::Event)
+          expect(event.get("_index")).not_to be_nil
+          expect(event.get("_id")).not_to be_nil
+          expect(event.get("_version")).not_to be_nil
+        end
+      end
+    end
+
+    context "#invalid ES|QL query" do
+      let(:config) do
+        super().merge("query" => "FROM undefined index | LIMIT 1")
+      end
+
+      it "doesn't produce events" do
+        queue = Queue.new
+        plugin.run(queue)
+        expect(queue.empty?).to eq(true)
+      end
+    end
+  end
+end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -76,6 +76,14 @@ describe LogStash::Inputs::Elasticsearch do
     shared_examples 'secured_elasticsearch' do
       it_behaves_like 'an elasticsearch index plugin'
 
+      let(:unauth_exception_class) do
+        begin
+          Elasticsearch::Transport::Transport::Errors::Unauthorized
+        rescue
+          Elastic::Transport::Transport::Errors::Unauthorized
+        end
+      end
+
       context "incorrect auth credentials" do
 
         let(:config) do
@@ -85,7 +93,7 @@ describe LogStash::Inputs::Elasticsearch do
         let(:queue) { [] }
 
         it "fails to run the plugin" do
-          expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+          expect { plugin.register }.to raise_error unauth_exception_class
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.21.2
+  version: 4.23.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-17 00:00:00.000000000 Z
+date: 2025-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -278,6 +278,8 @@ files:
 - lib/logstash/helpers/loggable_try.rb
 - lib/logstash/inputs/elasticsearch.rb
 - lib/logstash/inputs/elasticsearch/aggregation.rb
+- lib/logstash/inputs/elasticsearch/cursor_tracker.rb
+- lib/logstash/inputs/elasticsearch/esql.rb
 - lib/logstash/inputs/elasticsearch/paginated_search.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
@@ -291,8 +293,11 @@ files:
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
 - spec/fixtures/test_certs/renew.sh
+- spec/inputs/cursor_tracker_spec.rb
+- spec/inputs/elasticsearch_esql_spec.rb
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
+- spec/inputs/integration/elasticsearch_esql_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 - spec/inputs/paginated_search_spec.rb
 homepage: https://elastic.co/logstash
@@ -330,7 +335,10 @@ test_files:
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
 - spec/fixtures/test_certs/renew.sh
+- spec/inputs/cursor_tracker_spec.rb
+- spec/inputs/elasticsearch_esql_spec.rb
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
+- spec/inputs/integration/elasticsearch_esql_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 - spec/inputs/paginated_search_spec.rb