logstash-input-elasticsearch 4.23.0 → 5.0.0

data/spec/fixtures/test_certs/es.chain.crt

@@ -1,38 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
-dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
-MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
-9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
-V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
-36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
-MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
-/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
-hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
-BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
-K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
-wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
-CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
-T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
-byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
-UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
-avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
-dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
-MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
-aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
-AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
-QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
-90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
-99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
-i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
-m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
-BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
-AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
-lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
-p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
-E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
-IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
-T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
------END CERTIFICATE-----

data/spec/fixtures/test_certs/renew.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-cd "$(dirname "$0")"
-
-openssl x509 -x509toreq -in ca.crt -copy_extensions copyall -signkey ca.key -out ca.csr
-openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
-openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
-
-openssl x509 -x509toreq -in es.crt -copy_extensions copyall -signkey es.key -out es.csr
-openssl x509 -req -copy_extensions copyall -days 365 -in es.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out es.crt && rm es.csr
-cat es.crt ca.crt > es.chain.crt
-
-# output ISO8601 timestamp to file
-date -Iseconds > GENERATED_AT

data/spec/inputs/cursor_tracker_spec.rb

@@ -1,72 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/devutils/rspec/shared_examples"
-require "logstash/inputs/elasticsearch"
-require "logstash/inputs/elasticsearch/cursor_tracker"
-
-describe LogStash::Inputs::Elasticsearch::CursorTracker do
-
-  let(:last_run_metadata_path) { Tempfile.new('cursor_tracker_testing').path }
-  let(:tracking_field_seed) { "1980-01-01T23:59:59.999999999Z" }
-  let(:options) do
-    {
-      :last_run_metadata_path => last_run_metadata_path,
-      :tracking_field => "my_field",
-      :tracking_field_seed => tracking_field_seed
-    }
-  end
-
-  subject { described_class.new(**options) }
-
-  it "creating a class works" do
-    expect(subject).to be_a described_class
-  end
-
-  describe "checkpoint_cursor" do
-    before(:each) do
-      subject.checkpoint_cursor(intermediate: false) # store seed value
-      [
-        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-03T23:59:59.999999999Z")) },
-        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-01T23:59:59.999999999Z")) },
-        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-02T23:59:59.999999999Z")) },
-      ].each(&:join)
-    end
-    context "when doing intermediate checkpoint" do
-      it "persists the smallest value" do
-        subject.checkpoint_cursor(intermediate: true)
-        expect(IO.read(last_run_metadata_path)).to eq("2025-01-01T23:59:59.999999999Z")
-      end
-    end
-    context "when doing non-intermediate checkpoint" do
-      it "persists the largest value" do
-        subject.checkpoint_cursor(intermediate: false)
-        expect(IO.read(last_run_metadata_path)).to eq("2025-01-03T23:59:59.999999999Z")
-      end
-    end
-  end
-
-  describe "inject_cursor" do
-    let(:new_value) { "2025-01-03T23:59:59.999999999Z" }
-    let(:fake_now) { "2026-09-19T23:59:59.999999999Z" }
-
-    let(:query) do
-      %q[
-      { "query": { "range": { "event.ingested": { "gt": :last_value, "lt": :present}}}, "sort": [ { "event.ingested": {"order": "asc", "format": "strict_date_optional_time_nanos", "numeric_type" : "date_nanos" } } ] }
-      ]
-    end
-
-    before(:each) do
-      subject.record_last_value(LogStash::Event.new("my_field" => new_value))
-      subject.checkpoint_cursor(intermediate: false)
-      allow(subject).to receive(:now_minus_30s).and_return(fake_now)
-    end
-
-    it "injects the value of the cursor into json query if it contains :last_value" do
-      expect(subject.inject_cursor(query)).to match(/#{new_value}/)
-    end
-
-    it "injects current time into json query if it contains :present" do
-      expect(subject.inject_cursor(query)).to match(/#{fake_now}/)
-    end
-  end
-end

data/spec/inputs/elasticsearch_esql_spec.rb

@@ -1,180 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/inputs/elasticsearch"
-require "elasticsearch"
-
-describe LogStash::Inputs::Elasticsearch::Esql do
-  let(:client) { instance_double(Elasticsearch::Client) }
-  let(:esql_client) { double("esql-client") }
-
-  let(:plugin) { instance_double(LogStash::Inputs::Elasticsearch, params: plugin_config, decorate_event: nil) }
-  let(:plugin_config) do
-    {
-      "query" => "FROM test-index | STATS count() BY field",
-      "retries" => 3
-    }
-  end
-  let(:esql_executor) { described_class.new(client, plugin) }
-
-  describe "#initialization" do
-    it "sets up the ESQL client with correct parameters" do
-      expect(esql_executor.instance_variable_get(:@query)).to eq(plugin_config["query"])
-      expect(esql_executor.instance_variable_get(:@retries)).to eq(plugin_config["retries"])
-      expect(esql_executor.instance_variable_get(:@target_field)).to eq(nil)
-    end
-  end
-
-  describe "#execution" do
-    let(:output_queue) { Queue.new }
-
-    context "when faces error while retrying" do
-      it "retries the given block the specified number of times" do
-        attempts = 0
-        result = esql_executor.retryable("Test Job") do
-          attempts += 1
-          raise StandardError if attempts < 3
-          "success"
-        end
-        expect(attempts).to eq(3)
-        expect(result).to eq("success")
-      end
-
-      it "returns false if the block fails all attempts" do
-        result = esql_executor.retryable("Test Job") do
-          raise StandardError
-        end
-        expect(result).to eq(false)
-      end
-    end
-
-    context "when executing chain of processes" do
-      let(:response) { { 'values' => [%w[foo bar]], 'columns' => [{ 'name' => 'a.b.1.d', 'type' => 'keyword' },
-                                                                  { 'name' => 'h_g.k$l.m.0', 'type' => 'keyword' }] } }
-
-      before do
-        allow(esql_executor).to receive(:retryable).and_yield
-        allow(client).to receive_message_chain(:esql, :query).and_return(response)
-      end
-
-      it "executes the ESQL query and processes the results" do
-        allow(response).to receive(:headers).and_return({})
-        esql_executor.do_run(output_queue, plugin_config["query"])
-        expect(output_queue.size).to eq(1)
-
-        event = output_queue.pop
-        expect(event.get('[a][b][1][d]')).to eq('foo')
-        expect(event.get('[h_g][k$l][m][0]')).to eq('bar')
-      end
-
-      it "logs a warning if the response contains a warning header" do
-        allow(response).to receive(:headers).and_return({ "warning" => "some warning" })
-        expect(esql_executor.logger).to receive(:warn).with("ES|QL executor received warning", { :warning_message => "some warning" })
-        esql_executor.do_run(output_queue, plugin_config["query"])
-      end
-
-      it "does not log a warning if the response does not contain a warning header" do
-        allow(response).to receive(:headers).and_return({})
-        expect(esql_executor.logger).not_to receive(:warn)
-        esql_executor.do_run(output_queue, plugin_config["query"])
-      end
-    end
-
-    describe "multiple rows in the result" do
-      let(:response) { { 'values' => rows, 'columns' => [{ 'name' => 'key.1', 'type' => 'keyword' },
-                                                         { 'name' => 'key.2', 'type' => 'keyword' }] } }
-
-      before do
-        allow(esql_executor).to receive(:retryable).and_yield
-        allow(client).to receive_message_chain(:esql, :query).and_return(response)
-        allow(response).to receive(:headers).and_return({})
-      end
-
-      context "when mapping" do
-        let(:rows) { [%w[foo bar], %w[hello world]] }
-
-        it "1:1 maps rows to events" do
-          esql_executor.do_run(output_queue, plugin_config["query"])
-          expect(output_queue.size).to eq(2)
-
-          event_1 = output_queue.pop
-          expect(event_1.get('[key][1]')).to eq('foo')
-          expect(event_1.get('[key][2]')).to eq('bar')
-
-          event_2 = output_queue.pop
-          expect(event_2.get('[key][1]')).to eq('hello')
-          expect(event_2.get('[key][2]')).to eq('world')
-        end
-      end
-
-      context "when partial nil values appear" do
-        let(:rows) { [[nil, "bar"], ["hello", nil]] }
-
-        it "ignores the nil values" do
-          esql_executor.do_run(output_queue, plugin_config["query"])
-          expect(output_queue.size).to eq(2)
-
-          event_1 = output_queue.pop
-          expect(event_1.get('[key][1]')).to eq(nil)
-          expect(event_1.get('[key][2]')).to eq('bar')
-
-          event_2 = output_queue.pop
-          expect(event_2.get('[key][1]')).to eq('hello')
-          expect(event_2.get('[key][2]')).to eq(nil)
-        end
-      end
-    end
-
-    context "when sub-elements occur in the result" do
-      let(:response) { {
-        'values' => [[50, 1, 100], [50, 0, 1000], [50, 9, 99999]],
-        'columns' =>
-          [
-            { 'name' => 'time', 'type' => 'long' },
-            { 'name' => 'time.min', 'type' => 'long' },
-            { 'name' => 'time.max', 'type' => 'long' },
-          ]
-      } }
-
-      before do
-        allow(esql_executor).to receive(:retryable).and_yield
-        allow(client).to receive_message_chain(:esql, :query).and_return(response)
-        allow(response).to receive(:headers).and_return({})
-      end
-
-      it "includes 1st depth elements into event" do
-        esql_executor.do_run(output_queue, plugin_config["query"])
-
-        expect(output_queue.size).to eq(3)
-        3.times do
-          event = output_queue.pop
-          expect(event.get('time')).to eq(50)
-          expect(event.get('[time][min]')).to eq(nil)
-          expect(event.get('[time][max]')).to eq(nil)
-        end
-      end
-    end
-  end
-
-  describe "#column spec" do
-    let(:valid_spec) { { 'name' => 'field.name', 'type' => 'keyword' } }
-    let(:column_spec) { LogStash::Inputs::Elasticsearch::ColumnSpec.new(valid_spec) }
-
-    context "when initializes" do
-      it "sets the name and type attributes" do
-        expect(column_spec.name).to eq("field.name")
-        expect(column_spec.type).to eq("keyword")
-      end
-
-      it "freezes the name and type attributes" do
-        expect(column_spec.name).to be_frozen
-        expect(column_spec.type).to be_frozen
-      end
-    end
-
-    context "when calls the field reference" do
-      it "returns the correct field reference format" do
-        expect(column_spec.field_reference).to eq("[field][name]")
-      end
-    end
-  end
-end if LOGSTASH_VERSION >= LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION

data/spec/inputs/integration/elasticsearch_esql_spec.rb

@@ -1,150 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/inputs/elasticsearch"
-require "elasticsearch"
-require_relative "../../../spec/es_helper"
-
-describe LogStash::Inputs::Elasticsearch, integration: true do
-
-  SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
-  ES_HOSTS = ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"]
-
-  let(:plugin) { described_class.new(config) }
-  let(:es_index) { "logstash-esql-integration-#{rand(1000)}" }
-  let(:test_documents) do
-    [
-      { "message" => "test message 1", "type" => "a", "count" => 1 },
-      { "message" => "test message 2", "type" => "a", "count" => 2 },
-      { "message" => "test message 3", "type" => "b", "count" => 3 },
-      { "message" => "test message 4", "type" => "b", "count" => 4 },
-      { "message" => "test message 5", "type" => "c", "count" => 5 }
-    ]
-  end
-  let(:config) do
-    {
-      "hosts" => ES_HOSTS,
-      "query_type" => "esql"
-    }
-  end
-  let(:es_client) do
-    Elasticsearch::Client.new(hosts: ES_HOSTS)
-  end
-
-  before(:all) do
-    is_ls_with_esql_supported_client = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
-    skip "LS version does not have ES client which supports ES|QL" unless is_ls_with_esql_supported_client
-
-    # Skip tests if ES version doesn't support ES||QL
-    es_client = Elasticsearch::Client.new(hosts: ES_HOSTS) # need to separately create since let isn't allowed in before(:context)
-    es_version_info = es_client.info["version"]
-    es_gem_version = Gem::Version.create(es_version_info["number"])
-    skip "ES version does not support ES|QL" if es_gem_version.nil? || es_gem_version < Gem::Version.create(LogStash::Inputs::Elasticsearch::ES_ESQL_SUPPORT_VERSION)
-  end
-
-  before(:each) do
-    # Create index with test documents
-    es_client.indices.create(index: es_index, body: {}) unless es_client.indices.exists?(index: es_index)
-
-    test_documents.each do |doc|
-      es_client.index(index: es_index, body: doc, refresh: true)
-    end
-  end
-
-  after(:each) do
-    es_client.indices.delete(index: es_index) if es_client.indices.exists?(index: es_index)
-  end
-
-  context "#run ES|QL queries" do
-
-    before do
-      stub_const("LOGSTASH_VERSION", LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
-      allow_any_instance_of(LogStash::Inputs::Elasticsearch).to receive(:exit_plugin?).and_return false, true
-    end
-
-    before(:each) do
-      plugin.register
-    end
-
-    shared_examples "ESQL query execution" do |expected_count|
-      it "correctly retrieves documents" do
-        queue = Queue.new
-        plugin.run(queue)
-
-        event_count = 0
-        expected_count.times do |i|
-          event = queue.pop
-          expect(event).to be_a(LogStash::Event)
-          event_count += 1
-        end
-        expect(event_count).to eq(expected_count)
-      end
-    end
-
-    context "#FROM query" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index} | SORT count")
-      end
-
-      include_examples "ESQL query execution", 5
-    end
-
-    context "#FROM query and WHERE clause" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index} | WHERE type == \"a\" | SORT count")
-      end
-
-      include_examples "ESQL query execution", 2
-    end
-
-    context "#STATS aggregation" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index} | STATS avg(count) BY type")
-      end
-
-      it "retrieves aggregated stats" do
-        queue = Queue.new
-        plugin.run(queue)
-        results = []
-        3.times do
-          event = queue.pop
-          expect(event).to be_a(LogStash::Event)
-          results << event.get("avg(count)")
-        end
-
-        expected_averages = [1.5, 3.5, 5.0]
-        expect(results.sort).to eq(expected_averages)
-      end
-    end
-
-    context "#METADATA" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index} METADATA _index, _id, _version | DROP message.keyword, type.keyword | SORT count")
-      end
-
-      it "includes document metadata" do
-        queue = Queue.new
-        plugin.run(queue)
-
-        5.times do
-          event = queue.pop
-          expect(event).to be_a(LogStash::Event)
-          expect(event.get("_index")).not_to be_nil
-          expect(event.get("_id")).not_to be_nil
-          expect(event.get("_version")).not_to be_nil
-        end
-      end
-    end
-
-    context "#invalid ES|QL query" do
-      let(:config) do
-        super().merge("query" => "FROM undefined index | LIMIT 1")
-      end
-
-      it "doesn't produce events" do
-        queue = Queue.new
-        plugin.run(queue)
-        expect(queue.empty?).to eq(true)
-      end
-    end
-  end
-end