logstash-filter-elasticsearch 3.19.0 → 4.0.0

data/spec/filters/elasticsearch_esql_spec.rb

@@ -1,211 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/filters/elasticsearch"
-
-describe LogStash::Filters::Elasticsearch::EsqlExecutor do
-  let(:client) { instance_double(LogStash::Filters::ElasticsearchClient) }
-  let(:logger) { double("logger") }
-  let(:plugin) { LogStash::Filters::Elasticsearch.new(plugin_config) }
-  let(:plugin_config) do
-    {
-      "query_type" => "esql",
-      "query" => "FROM test-index | STATS count() BY field | LIMIT 10"
-    }
-  end
-  let(:esql_executor) { described_class.new(plugin, logger) }
-
-  context "when initializes" do
-    it "sets up the ESQL executor with correct parameters" do
-      allow(logger).to receive(:debug)
-      allow(logger).to receive(:warn)
-      expect(esql_executor.instance_variable_get(:@query)).to eq(plugin_config["query"])
-      expect(esql_executor.instance_variable_get(:@referenced_params)).to eq({})
-      expect(esql_executor.instance_variable_get(:@static_params)).to eq([])
-      expect(esql_executor.instance_variable_get(:@tag_on_failure)).to eq(["_elasticsearch_lookup_failure"])
-    end
-  end
-
-  context "when processes" do
-    let(:plugin_config) {
-      super()
-        .merge(
-          {
-            "query" => "FROM my-index | WHERE field = ?foo | LIMIT 5",
-            "query_params" => { "foo" => "[bar]" }
-          })
-    }
-    let(:event) { LogStash::Event.new({}) }
-    let(:response) {
-      {
-        'values' => [["foo", "bar", nil]],
-        'columns' => [{ 'name' => 'id', 'type' => 'keyword' }, { 'name' => 'val', 'type' => 'keyword' }, { 'name' => 'odd', 'type' => 'keyword' }]
-      }
-    }
-
-    before do
-      allow(logger).to receive(:debug)
-      allow(logger).to receive(:warn)
-    end
-
-    it "resolves parameters" do
-      expect(event).to receive(:get).with("[bar]").and_return("resolved_value")
-      resolved_params = esql_executor.send(:resolve_parameters, event)
-      expect(resolved_params).to include("foo" => "resolved_value")
-    end
-
-    it "executes the query with resolved parameters" do
-      allow(logger).to receive(:debug)
-      expect(event).to receive(:get).with("[bar]").and_return("resolved_value")
-      expect(client).to receive(:esql_query).with(
-        { body: { query: plugin_config["query"], params: [{ "foo" => "resolved_value" }] }, format: 'json', drop_null_columns: true, })
-      resolved_params = esql_executor.send(:resolve_parameters, event)
-      esql_executor.send(:execute_query, client, resolved_params)
-    end
-
-    it "informs warning if received warning" do
-      allow(response).to receive(:headers).and_return({ "warning" => "some warning" })
-      expect(logger).to receive(:warn).with("ES|QL executor received warning", { :message => "some warning" })
-      esql_executor.send(:inform_warning, response)
-    end
-
-    it "processes the response and adds metadata" do
-      expect(event).to receive(:set).with("[@metadata][total_values]", 1)
-      # [id], [val] aren't resolved via sprintf, use as it is
-      expect(event).to receive(:set).with("[id]", "foo")
-      expect(event).to receive(:set).with("[val]", "bar")
-      esql_executor.send(:process_response, event, response)
-    end
-
-    it "executes chain of processes" do
-      allow(plugin).to receive(:decorate)
-      allow(logger).to receive(:debug)
-      allow(response).to receive(:headers).and_return({})
-      expect(client).to receive(:esql_query).with(
-        {
-          body: { query: plugin_config["query"], params: [{"foo"=>"resolve_me"}] },
-          format: 'json',
-          drop_null_columns: true,
-        }).and_return(response)
-
-      event = LogStash::Event.new({ "hello" => "world", "bar" => "resolve_me" })
-      expect { esql_executor.process(client, event) }.to_not raise_error
-      expect(event.get("[@metadata][total_values]")).to eq(1)
-      expect(event.get("hello")).to eq("world")
-      expect(event.get("val")).to eq("bar")
-      expect(event.get("odd")).to be_nil # filters out non-exist fields
-    end
-
-    it "tags on plugin failures" do
-      expect(event).to receive(:get).with("[bar]").and_raise("Event#get Invalid FieldReference error")
-
-      expect(logger).to receive(:error).with("Failed to process ES|QL filter", exception: instance_of(RuntimeError))
-      expect(event).to receive(:tag).with("_elasticsearch_lookup_failure")
-      esql_executor.process(client, event)
-    end
-
-    it "tags on query execution failures" do
-      allow(logger).to receive(:debug)
-      allow(client).to receive(:esql_query).and_raise("Query execution error")
-
-      expect(logger).to receive(:error).with("Failed to process ES|QL filter", exception: instance_of(RuntimeError))
-      expect(event).to receive(:tag).with("_elasticsearch_lookup_failure")
-      esql_executor.process(client, event)
-    end
-
-    describe "#target" do
-      let(:event) { LogStash::Event.new({ "hello" => "world", "bar" => "resolve_me" }) }
-      let(:response) {
-        super().merge({ 'values' => [["foo", "bar", nil], %w[hello again world], %w[another value here]] })
-      }
-      before(:each) do
-        expect(client).to receive(:esql_query).with(any_args).and_return(response)
-        allow(plugin).to receive(:decorate)
-        allow(logger).to receive(:debug)
-        allow(response).to receive(:headers).and_return({})
-      end
-
-      context "when specified" do
-        let(:plugin_config) {
-          super().merge({ "target" => "my-target" })
-        }
-
-        it "sets all query results into event" do
-          expected_result = [
-            {"id"=>"foo", "val"=>"bar"},
-            {"id"=>"hello", "val"=>"again", "odd"=>"world"},
-            {"id"=>"another", "val"=>"value", "odd"=>"here"}
-          ]
-          expect { esql_executor.process(client, event) }.to_not raise_error
-          expect(event.get("[@metadata][total_values]")).to eq(3)
-          expect(event.get("my-target").size).to eq(3)
-          expect(event.get("my-target")).to eq(expected_result)
-        end
-      end
-
-      context "when not specified" do
-        shared_examples "first result into the event" do
-          it "sets" do
-            expect { esql_executor.process(client, event) }.to_not raise_error
-            expect(event.get("[@metadata][total_values]")).to eq(3)
-            expect(event.get("id")).to eq("foo")
-            expect(event.get("val")).to eq("bar")
-            expect(event.get("odd")).to eq(nil)
-          end
-        end
-        context "when limit is included in the query" do
-          let(:plugin_config) {
-            super().merge({ "query" => "FROM my-index | LIMIT 555" })
-          }
-          it_behaves_like "first result into the event"
-        end
-
-        context "when limit is not included in the query" do
-          let(:plugin_config) {
-            super().merge({ "query" => "FROM my-index" })
-          }
-          it_behaves_like "first result into the event"
-        end
-      end
-    end
-  end
-
-  describe "#query placeholders" do
-    before(:each) do
-      allow(logger).to receive(:debug)
-      allow(logger).to receive(:warn)
-      plugin.send(:validate_esql_query_and_params!)
-    end
-
-    context "when `query_params` is an Array contains {key => val} entries" do
-      let(:plugin_config) {
-        super()
-          .merge(
-            {
-              "query" => "FROM my-index | LIMIT 1",
-              "query_params" => [{ "a" => "b" }, { "c" => "[b]" }, { "e" => 1 }, { "f" => "[g]" }],
-            })
-      }
-
-      it "separates references and static params at initialization" do
-        expect(esql_executor.instance_variable_get(:@referenced_params)).to eq({"c" => "[b]", "f" => "[g]"})
-        expect(esql_executor.instance_variable_get(:@static_params)).to eq([{"a" => "b"},  {"e" => 1}])
-      end
-    end
-
-    context "when `query_params` is a Hash" do
-      let(:plugin_config) {
-        super()
-          .merge(
-            {
-              "query" => "FROM my-index | LIMIT 1",
-              "query_params" => { "a" => "b", "c" => "[b]", "e" => 1, "f" => "[g]" },
-            })
-      }
-
-      it "separates references and static params at initialization" do
-        expect(esql_executor.instance_variable_get(:@referenced_params)).to eq({"c" => "[b]", "f" => "[g]"})
-        expect(esql_executor.instance_variable_get(:@static_params)).to eq([{"a" => "b"},  {"e" => 1}])
-      end
-    end
-  end
-end if LOGSTASH_VERSION >= LogStash::Filters::Elasticsearch::LS_ESQL_SUPPORT_VERSION

data/spec/filters/integration/elasticsearch_esql_spec.rb

@@ -1,167 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/filters/elasticsearch"
-require "elasticsearch"
-require_relative "../../../spec/es_helper"
-
-describe LogStash::Filters::Elasticsearch, integration: true do
-
-  ELASTIC_SECURITY_ENABLED = ENV['ELASTIC_SECURITY_ENABLED'].eql? 'true'
-  SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
-  ES_HOSTS = ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"]
-  CA_PATH = File.expand_path('../fixtures/test_certs/ca.crt', File.dirname(__FILE__))
-
-  let(:plugin) { described_class.new(config) }
-  let(:es_index) { "es-filter-plugin-esql-integration-#{rand(1000)}" }
-  let(:test_documents) do
-    [
-      { "message" => "test message 1", "type" => "a", "count" => 1 },
-      { "message" => "test message 2", "type" => "a", "count" => 2 },
-      { "message" => "test message 3", "type" => "b", "count" => 3 },
-      { "message" => "test message 4", "type" => "b", "count" => 4 },
-      { "message" => "test message 5", "type" => "c", "count" => 5 },
-      { "message" => "odd test message", "type" => "t" }
-    ]
-  end
-
-  let(:base_config) do
-    {
-      "query_type" => "esql",
-      "hosts" => ES_HOSTS,
-      "ssl_enabled" => SECURE_INTEGRATION
-    }
-  end
-
-  let(:credentials) do
-    if SECURE_INTEGRATION
-      { 'user' => 'tests', 'password' => 'Tests123' }
-    else
-      { 'user' => 'elastic', 'password' => ENV['ELASTIC_PASSWORD'] }
-    end
-  end
-
-  let(:config) do
-    config = ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config
-    config = { 'ssl_certificate_authorities' => CA_PATH }.merge(config) if SECURE_INTEGRATION
-    config
-  end
-
-  let(:event) { LogStash::Event.new({}) }
-
-  def es_client
-    @es_client ||= begin
-      user = SECURE_INTEGRATION ? 'tests' : 'elastic'
-      password = SECURE_INTEGRATION ? 'Tests123' : ENV['ELASTIC_PASSWORD']
-
-      es_client_config = { hosts: ES_HOSTS }
-      es_client_config = es_client_config.merge({ user: user, password: password }) if ELASTIC_SECURITY_ENABLED || SECURE_INTEGRATION
-      es_client_config = es_client_config.merge({ transport_options: { ssl: { ca_path: CA_PATH, verify: false }}}) if SECURE_INTEGRATION
-
-      Elasticsearch::Client.new(es_client_config)
-    end
-  end
-
-  before(:all) do
-    is_ls_with_esql_supported_client = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(LogStash::Filters::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
-    # Skip tests if an ES version doesn't support ES|QL
-    skip "LS version does not have ES client which supports ES|QL" unless is_ls_with_esql_supported_client
-
-    es_version_info = es_client.info["version"]
-    es_gem_version = Gem::Version.create(es_version_info["number"])
-    skip "ES version does not support ES|QL" if es_gem_version.nil? || es_gem_version < Gem::Version.create(LogStash::Filters::Elasticsearch::ES_ESQL_SUPPORT_VERSION)
-  end
-
-  before(:each) do
-    # Create index with test documents
-    es_client.indices.create(index: es_index, body: {}) unless es_client.indices.exists?(index: es_index)
-
-    test_documents.each do |doc|
-      es_client.index(index: es_index, body: doc, refresh: true)
-    end
-  end
-
-  after(:each) do
-    es_client.indices.delete(index: es_index) if es_client.indices.exists?(index: es_index)
-  end
-
-  describe "run ES|QL queries" do
-
-    before do
-      stub_const("LOGSTASH_VERSION", LogStash::Filters::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
-    end
-
-    before(:each) do
-      plugin.register
-    end
-
-    shared_examples "ESQL query execution" do |expected_count, fields|
-      it "processes the event" do
-        plugin.filter(event)
-        expect(event.get("[@metadata][total_values]")).to eq(expected_count)
-        fields&.each do | field |
-          expect(event.get(field)).not_to be(nil)
-        end
-      end
-    end
-
-    describe "with simple FROM query with LIMIT" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index} | LIMIT 99")
-      end
-
-      include_examples "ESQL query execution", 6
-    end
-
-    describe "with simple FROM and WHERE query combinations" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index} | WHERE type==\"b\" | LIMIT 99")
-      end
-
-      include_examples "ESQL query execution", 2
-    end
-
-    describe "with query params" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index} | WHERE type==?type", "query_params" => { "type" => "b" })
-      end
-
-      include_examples "ESQL query execution", 2
-    end
-
-    describe "when invalid query used" do
-      let(:config) do
-        super().merge("query" => "FROM undefined index | LIMIT 1")
-      end
-
-      it "tags on failure" do
-        plugin.filter(event)
-        expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
-      end
-    end
-
-    describe "when field enrichment requested" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index} | WHERE type==\"b\" | LIMIT 99")
-      end
-
-      include_examples "ESQL query execution", 2,  %w[message count]
-    end
-
-    describe "when non-exist field value appear" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index}", "target" => "target_field")
-      end
-
-      it "processes the event" do
-        plugin.filter(event)
-        expect(event.get("[@metadata][total_values]")).to eq(6)
-        expect(event.get("target_field").size).to eq(6)
-        values = event.get("target_field")
-        counts = values.count { |entry| entry.key?("count") }
-        messages = values.count { |entry| entry.key?("message") }
-        expect(counts).to eq(5)
-        expect(messages).to eq(6)
-      end
-    end
-  end
-end