logstash-input-elasticsearch 4.22.0 → 4.23.1

data/spec/fixtures/test_certs/es.chain.crt

@@ -1,7 +1,7 @@
 -----BEGIN CERTIFICATE-----
 MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
-dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
-MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNjAxMjMx
+NjM4MjVaFw0yNzAxMjMxNjM4MjVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
 9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
 V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
 36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
@@ -10,17 +10,17 @@ MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
 hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
 BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
 K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
-wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
-CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
-T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
-byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
-UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
-avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
+wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQCDFHoe8ntx/D69gb8uVQhUDI2ERb0Hocbh
+BpQVidsKEv220zmZW3O9PX0aN4j3e4HUeRNSdLmtN+sXJ1xCMjHlzA5z5wYJOk2y
+4B/eCOzHoHfvR+l5NUSemZ5mk8V+gThGydPmLZBN3afjzd3NJ7NVaCA8cyKwe2jr
+AMrIdGL5usoBSdbd7Ww0nWhLhqxmmV79+orCwAPJxkzsgFDK+vfIw4VHERXNoIPe
+WKVBGpUCFPSfosLo9fSziNDXL71CrWZlPXt3Otip4QKQV5sZI2zxm1JtHdHX6Ul1
+X7xkcUR5cOF0InxefGl9VHIZTYC5kSzAz95WV068j2Jxgu28GRP5
 -----END CERTIFICATE-----
 -----BEGIN CERTIFICATE-----
 MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
-dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
-MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNjAxMjMx
+NjM4MjVaFw0yNzAxMjMxNjM4MjVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
 aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
 AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
 QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
@@ -29,10 +29,10 @@ QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
 i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
 m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
 BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
-AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
-lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
-p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
-E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
-IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
-T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
+AQsFAAOCAQEAgZPz/e29AepHZu8uuO6+75b6Uf88m8x5m7XL13m9+vTTSqLvCiBw
+JCf5caO73iOx8AlTT2iSlKPAS/7ogik5anZziqweHmBakb+HNQcUB4Vv4VBo5Ai3
+SdOm28uu4EznVLUILPUHl4FQGxhn6ba1tLEWgxlD/ynBtfcmh9UFtBqIawrpAArv
+TKwGBSkcSxh/yOIo+qbyBh9EyxxlGW/Z08m9qjNqHATDxCelcTsQLhCHkvNShR4j
+v1sFw5XQ8H3zQU6w4WzV81A3K8erIU8WnvdisXLk0T1VB5KURy4+UDyw44tOAPD2
+vBq3gYLgcVEMYVfOWCzDy7LS49leXRzAOA==
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/es.crt

@@ -1,7 +1,7 @@
 -----BEGIN CERTIFICATE-----
 MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
-dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
-MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNjAxMjMx
+NjM4MjVaFw0yNzAxMjMxNjM4MjVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
 9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
 V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
 36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
@@ -10,10 +10,10 @@ MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
 hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
 BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
 K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
-wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
-CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
-T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
-byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
-UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
-avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
+wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQCDFHoe8ntx/D69gb8uVQhUDI2ERb0Hocbh
+BpQVidsKEv220zmZW3O9PX0aN4j3e4HUeRNSdLmtN+sXJ1xCMjHlzA5z5wYJOk2y
+4B/eCOzHoHfvR+l5NUSemZ5mk8V+gThGydPmLZBN3afjzd3NJ7NVaCA8cyKwe2jr
+AMrIdGL5usoBSdbd7Ww0nWhLhqxmmV79+orCwAPJxkzsgFDK+vfIw4VHERXNoIPe
+WKVBGpUCFPSfosLo9fSziNDXL71CrWZlPXt3Otip4QKQV5sZI2zxm1JtHdHX6Ul1
+X7xkcUR5cOF0InxefGl9VHIZTYC5kSzAz95WV068j2Jxgu28GRP5
 -----END CERTIFICATE-----

data/spec/inputs/elasticsearch_esql_spec.rb

@@ -0,0 +1,180 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/elasticsearch"
+require "elasticsearch"
+
+describe LogStash::Inputs::Elasticsearch::Esql do
+  let(:client) { instance_double(Elasticsearch::Client) }
+  let(:esql_client) { double("esql-client") }
+
+  let(:plugin) { instance_double(LogStash::Inputs::Elasticsearch, params: plugin_config, decorate_event: nil) }
+  let(:plugin_config) do
+    {
+      "query" => "FROM test-index | STATS count() BY field",
+      "retries" => 3
+    }
+  end
+  let(:esql_executor) { described_class.new(client, plugin) }
+
+  describe "#initialization" do
+    it "sets up the ESQL client with correct parameters" do
+      expect(esql_executor.instance_variable_get(:@query)).to eq(plugin_config["query"])
+      expect(esql_executor.instance_variable_get(:@retries)).to eq(plugin_config["retries"])
+      expect(esql_executor.instance_variable_get(:@target_field)).to eq(nil)
+    end
+  end
+
+  describe "#execution" do
+    let(:output_queue) { Queue.new }
+
+    context "when faces error while retrying" do
+      it "retries the given block the specified number of times" do
+        attempts = 0
+        result = esql_executor.retryable("Test Job") do
+          attempts += 1
+          raise StandardError if attempts < 3
+          "success"
+        end
+        expect(attempts).to eq(3)
+        expect(result).to eq("success")
+      end
+
+      it "returns false if the block fails all attempts" do
+        result = esql_executor.retryable("Test Job") do
+          raise StandardError
+        end
+        expect(result).to eq(false)
+      end
+    end
+
+    context "when executing chain of processes" do
+      let(:response) { { 'values' => [%w[foo bar]], 'columns' => [{ 'name' => 'a.b.1.d', 'type' => 'keyword' },
+                                                                  { 'name' => 'h_g.k$l.m.0', 'type' => 'keyword' }] } }
+
+      before do
+        allow(esql_executor).to receive(:retryable).and_yield
+        allow(client).to receive_message_chain(:esql, :query).and_return(response)
+      end
+
+      it "executes the ESQL query and processes the results" do
+        allow(response).to receive(:headers).and_return({})
+        esql_executor.do_run(output_queue, plugin_config["query"])
+        expect(output_queue.size).to eq(1)
+
+        event = output_queue.pop
+        expect(event.get('[a][b][1][d]')).to eq('foo')
+        expect(event.get('[h_g][k$l][m][0]')).to eq('bar')
+      end
+
+      it "logs a warning if the response contains a warning header" do
+        allow(response).to receive(:headers).and_return({ "warning" => "some warning" })
+        expect(esql_executor.logger).to receive(:warn).with("ES|QL executor received warning", { :warning_message => "some warning" })
+        esql_executor.do_run(output_queue, plugin_config["query"])
+      end
+
+      it "does not log a warning if the response does not contain a warning header" do
+        allow(response).to receive(:headers).and_return({})
+        expect(esql_executor.logger).not_to receive(:warn)
+        esql_executor.do_run(output_queue, plugin_config["query"])
+      end
+    end
+
+    describe "multiple rows in the result" do
+      let(:response) { { 'values' => rows, 'columns' => [{ 'name' => 'key.1', 'type' => 'keyword' },
+                                                         { 'name' => 'key.2', 'type' => 'keyword' }] } }
+
+      before do
+        allow(esql_executor).to receive(:retryable).and_yield
+        allow(client).to receive_message_chain(:esql, :query).and_return(response)
+        allow(response).to receive(:headers).and_return({})
+      end
+
+      context "when mapping" do
+        let(:rows) { [%w[foo bar], %w[hello world]] }
+
+        it "1:1 maps rows to events" do
+          esql_executor.do_run(output_queue, plugin_config["query"])
+          expect(output_queue.size).to eq(2)
+
+          event_1 = output_queue.pop
+          expect(event_1.get('[key][1]')).to eq('foo')
+          expect(event_1.get('[key][2]')).to eq('bar')
+
+          event_2 = output_queue.pop
+          expect(event_2.get('[key][1]')).to eq('hello')
+          expect(event_2.get('[key][2]')).to eq('world')
+        end
+      end
+
+      context "when partial nil values appear" do
+        let(:rows) { [[nil, "bar"], ["hello", nil]] }
+
+        it "ignores the nil values" do
+          esql_executor.do_run(output_queue, plugin_config["query"])
+          expect(output_queue.size).to eq(2)
+
+          event_1 = output_queue.pop
+          expect(event_1.get('[key][1]')).to eq(nil)
+          expect(event_1.get('[key][2]')).to eq('bar')
+
+          event_2 = output_queue.pop
+          expect(event_2.get('[key][1]')).to eq('hello')
+          expect(event_2.get('[key][2]')).to eq(nil)
+        end
+      end
+    end
+
+    context "when sub-elements occur in the result" do
+      let(:response) { {
+        'values' => [[50, 1, 100], [50, 0, 1000], [50, 9, 99999]],
+        'columns' =>
+          [
+            { 'name' => 'time', 'type' => 'long' },
+            { 'name' => 'time.min', 'type' => 'long' },
+            { 'name' => 'time.max', 'type' => 'long' },
+          ]
+      } }
+
+      before do
+        allow(esql_executor).to receive(:retryable).and_yield
+        allow(client).to receive_message_chain(:esql, :query).and_return(response)
+        allow(response).to receive(:headers).and_return({})
+      end
+
+      it "includes 1st depth elements into event" do
+        esql_executor.do_run(output_queue, plugin_config["query"])
+
+        expect(output_queue.size).to eq(3)
+        3.times do
+          event = output_queue.pop
+          expect(event.get('time')).to eq(50)
+          expect(event.get('[time][min]')).to eq(nil)
+          expect(event.get('[time][max]')).to eq(nil)
+        end
+      end
+    end
+  end
+
+  describe "#column spec" do
+    let(:valid_spec) { { 'name' => 'field.name', 'type' => 'keyword' } }
+    let(:column_spec) { LogStash::Inputs::Elasticsearch::ColumnSpec.new(valid_spec) }
+
+    context "when initializes" do
+      it "sets the name and type attributes" do
+        expect(column_spec.name).to eq("field.name")
+        expect(column_spec.type).to eq("keyword")
+      end
+
+      it "freezes the name and type attributes" do
+        expect(column_spec.name).to be_frozen
+        expect(column_spec.type).to be_frozen
+      end
+    end
+
+    context "when calls the field reference" do
+      it "returns the correct field reference format" do
+        expect(column_spec.field_reference).to eq("[field][name]")
+      end
+    end
+  end
+end if LOGSTASH_VERSION >= LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION

data/spec/inputs/elasticsearch_spec.rb

@@ -810,14 +810,50 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
 
       context "with ssl" do
-        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl_enabled" => true }) }
+        let(:api_key_value) { nil }
+        let(:config) { super().merge("ssl_enabled" => true, 'api_key' => LogStash::Util::Password.new(api_key_value)) }
+        let(:encoded_api_key) { Base64.strict_encode64('foo:bar') }
 
-        it "should set authorization" do
-          plugin.register
-          client = plugin.send(:client)
-          auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+        shared_examples "a plugin that sets the ApiKey authorization header" do
+          it "correctly sets the Authorization header" do
+            plugin.register
+            client = plugin.send(:client)
+            auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+
+            expect(auth_header).to eql("ApiKey #{encoded_api_key}")
+          end
+        end
+
+        context "with a non-encoded API key" do
+          let(:api_key_value) { "foo:bar" }
+          it_behaves_like "a plugin that sets the ApiKey authorization header"
+        end
 
-          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
+        context "with an encoded API key" do
+          let(:api_key_value) { encoded_api_key }
+          it_behaves_like "a plugin that sets the ApiKey authorization header"
+        end
+
+        context "with an Elastic Cloud API key (essu_ prefix)" do
+          # The suffix is intentionally not canonical base64: a Cloud key is opaque
+          # and must be forwarded verbatim regardless of its payload encoding.
+          let(:api_key_value) { "essu_VFZGblZreFhTekJ4ZDB4M2NHUnZRMEU2YzNWd1pYSnpaV055WlhRPQ==AAAAAAAA" }
+
+          it "sets the Authorization header verbatim without re-encoding" do
+            plugin.register
+            client = plugin.send(:client)
+            auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+
+            expect(auth_header).to eql("ApiKey #{api_key_value}")
+          end
+        end
+
+        context "with an unrecognized api_key format" do
+          let(:api_key_value) { "not-a-valid-key" }
+
+          it "fails registration with a configuration error" do
+            expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Invalid api_key format/)
+          end
         end
 
         context 'user also set' do
@@ -1357,4 +1393,129 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
+  describe "#ESQL" do
+    let(:config) do
+      {
+        "query" => "FROM test-index | STATS count() BY field",
+        "query_type" => "esql",
+        "retries" => 3
+      }
+    end
+    let(:es_version) { LogStash::Inputs::Elasticsearch::ES_ESQL_SUPPORT_VERSION }
+    let(:ls_version) { LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION }
+
+    before(:each) do
+      stub_const("LOGSTASH_VERSION", ls_version)
+    end
+
+    describe "#initialize" do
+      it "sets up the ESQL client with correct parameters" do
+        expect(plugin.instance_variable_get(:@query_type)).to eq(config["query_type"])
+        expect(plugin.instance_variable_get(:@query)).to eq(config["query"])
+        expect(plugin.instance_variable_get(:@retries)).to eq(config["retries"])
+      end
+    end
+
+    describe "#register" do
+      before(:each) do
+        Elasticsearch::Client.send(:define_method, :ping) { }
+        allow_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(cluster_info)
+      end
+      it "creates ES|QL executor" do
+        plugin.register
+        expect(plugin.instance_variable_get(:@query_executor)).to be_an_instance_of(LogStash::Inputs::Elasticsearch::Esql)
+      end
+    end
+
+    describe "#validation" do
+
+      describe "LS version" do
+        context "when compatible" do
+
+          it "does not raise an error" do
+            expect { plugin.send(:validate_ls_version_for_esql_support!) }.not_to raise_error
+          end
+        end
+
+        context "when incompatible" do
+          before(:each) do
+            stub_const("LOGSTASH_VERSION", "8.10.0")
+          end
+
+          it "raises a runtime error" do
+            expect { plugin.send(:validate_ls_version_for_esql_support!) }
+              .to raise_error(RuntimeError, /Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{ls_version}/)
+          end
+        end
+      end
+
+      describe "ES version" do
+        before(:each) do
+          allow(plugin).to receive(:es_version).and_return("8.10.5")
+        end
+
+        context "when incompatible" do
+          it "raises a runtime error" do
+            expect { plugin.send(:validate_es_for_esql_support!) }
+              .to raise_error(RuntimeError, /Connected Elasticsearch 8.10.5 version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{es_version} version./)
+          end
+        end
+      end
+
+      context "ES|QL query and DSL params used together" do
+        let(:config) {
+          super().merge({
+            "index" => "my-index",
+            "size" => 1,
+            "slices" => 1,
+            "search_api" => "auto",
+            "docinfo" => true,
+            "docinfo_target" => "[@metadata][docinfo]",
+            "docinfo_fields" => ["_index"],
+            "response_type" => "hits",
+            "tracking_field" => "[@metadata][tracking]"
+          })}
+
+        it "raises a config error" do
+          mixed_fields = %w[index size slices docinfo_fields response_type tracking_field]
+          expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Configured #{mixed_fields} params are not allowed while using ES|QL query/)
+        end
+      end
+
+      describe "ES|QL query"  do
+        context "when query is valid" do
+          it "does not raise an error" do
+            expect { plugin.send(:validate_esql_query!) }.not_to raise_error
+          end
+        end
+
+        context "when query is empty" do
+          let(:config) do
+            {
+              "query" => " "
+            }
+          end
+
+          it "raises a configuration error" do
+            expect { plugin.send(:validate_esql_query!) }
+              .to raise_error(LogStash::ConfigurationError, /`query` cannot be empty/)
+          end
+        end
+
+        context "when query doesn't align with ES syntax" do
+          let(:config) do
+            {
+              "query" => "RANDOM query"
+            }
+          end
+
+          it "raises a configuration error" do
+            source_commands = %w[FROM ROW SHOW]
+            expect { plugin.send(:validate_esql_query!) }
+              .to raise_error(LogStash::ConfigurationError, "`query` needs to start with any of #{source_commands}")
+          end
+        end
+      end
+    end
+  end
 end

data/spec/inputs/integration/elasticsearch_esql_spec.rb

@@ -0,0 +1,150 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/elasticsearch"
+require "elasticsearch"
+require_relative "../../../spec/es_helper"
+
+describe LogStash::Inputs::Elasticsearch, integration: true do
+
+  SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
+  ES_HOSTS = ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"]
+
+  let(:plugin) { described_class.new(config) }
+  let(:es_index) { "logstash-esql-integration-#{rand(1000)}" }
+  let(:test_documents) do
+    [
+      { "message" => "test message 1", "type" => "a", "count" => 1 },
+      { "message" => "test message 2", "type" => "a", "count" => 2 },
+      { "message" => "test message 3", "type" => "b", "count" => 3 },
+      { "message" => "test message 4", "type" => "b", "count" => 4 },
+      { "message" => "test message 5", "type" => "c", "count" => 5 }
+    ]
+  end
+  let(:config) do
+    {
+      "hosts" => ES_HOSTS,
+      "query_type" => "esql"
+    }
+  end
+  let(:es_client) do
+    Elasticsearch::Client.new(hosts: ES_HOSTS)
+  end
+
+  before(:all) do
+    is_ls_with_esql_supported_client = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
+    skip "LS version does not have ES client which supports ES|QL" unless is_ls_with_esql_supported_client
+
+    # Skip tests if ES version doesn't support ES||QL
+    es_client = Elasticsearch::Client.new(hosts: ES_HOSTS) # need to separately create since let isn't allowed in before(:context)
+    es_version_info = es_client.info["version"]
+    es_gem_version = Gem::Version.create(es_version_info["number"])
+    skip "ES version does not support ES|QL" if es_gem_version.nil? || es_gem_version < Gem::Version.create(LogStash::Inputs::Elasticsearch::ES_ESQL_SUPPORT_VERSION)
+  end
+
+  before(:each) do
+    # Create index with test documents
+    es_client.indices.create(index: es_index, body: {}) unless es_client.indices.exists?(index: es_index)
+
+    test_documents.each do |doc|
+      es_client.index(index: es_index, body: doc, refresh: true)
+    end
+  end
+
+  after(:each) do
+    es_client.indices.delete(index: es_index) if es_client.indices.exists?(index: es_index)
+  end
+
+  context "#run ES|QL queries" do
+
+    before do
+      stub_const("LOGSTASH_VERSION", LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
+      allow_any_instance_of(LogStash::Inputs::Elasticsearch).to receive(:exit_plugin?).and_return false, true
+    end
+
+    before(:each) do
+      plugin.register
+    end
+
+    shared_examples "ESQL query execution" do |expected_count|
+      it "correctly retrieves documents" do
+        queue = Queue.new
+        plugin.run(queue)
+
+        event_count = 0
+        expected_count.times do |i|
+          event = queue.pop
+          expect(event).to be_a(LogStash::Event)
+          event_count += 1
+        end
+        expect(event_count).to eq(expected_count)
+      end
+    end
+
+    context "#FROM query" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | SORT count")
+      end
+
+      include_examples "ESQL query execution", 5
+    end
+
+    context "#FROM query and WHERE clause" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | WHERE type == \"a\" | SORT count")
+      end
+
+      include_examples "ESQL query execution", 2
+    end
+
+    context "#STATS aggregation" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | STATS avg(count) BY type")
+      end
+
+      it "retrieves aggregated stats" do
+        queue = Queue.new
+        plugin.run(queue)
+        results = []
+        3.times do
+          event = queue.pop
+          expect(event).to be_a(LogStash::Event)
+          results << event.get("avg(count)")
+        end
+
+        expected_averages = [1.5, 3.5, 5.0]
+        expect(results.sort).to eq(expected_averages)
+      end
+    end
+
+    context "#METADATA" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} METADATA _index, _id, _version | DROP message.keyword, type.keyword | SORT count")
+      end
+
+      it "includes document metadata" do
+        queue = Queue.new
+        plugin.run(queue)
+
+        5.times do
+          event = queue.pop
+          expect(event).to be_a(LogStash::Event)
+          expect(event.get("_index")).not_to be_nil
+          expect(event.get("_id")).not_to be_nil
+          expect(event.get("_version")).not_to be_nil
+        end
+      end
+    end
+
+    context "#invalid ES|QL query" do
+      let(:config) do
+        super().merge("query" => "FROM undefined index | LIMIT 1")
+      end
+
+      it "doesn't produce events" do
+        queue = Queue.new
+        plugin.run(queue)
+        expect(queue.empty?).to eq(true)
+      end
+    end
+  end
+end

data/version

@@ -0,0 +1 @@
+4.23.1