logstash-input-elasticsearch 4.23.0 → 5.0.0

data/spec/inputs/elasticsearch_spec.rb

@@ -21,13 +21,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   let(:es_version) { "7.5.0" }
   let(:cluster_info) { {"version" => {"number" => es_version, "build_flavor" => build_flavor}, "tagline" => "You Know, for Search"} }
 
-  def elastic_ruby_v8_client_available?
-    Elasticsearch::Transport
-    false
-  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
-    true
-  end
-
   before(:each) do
     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
     allow_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(cluster_info)
@@ -65,6 +58,19 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
     end
 
+    describe 'handling obsolete settings' do
+      [{:name => 'ssl', :replacement => 'ssl_enabled', :sample_value => true},
+       {:name => 'ca_file', :replacement => 'ssl_certificate_authorities', :sample_value => 'spec/fixtures/test_certs/ca.crt'},
+       {:name => 'ssl_certificate_verification', :replacement => 'ssl_verification_mode', :sample_value => false }].each do | obsolete_setting|
+        context "with obsolete #{obsolete_setting[:name]}" do
+          let (:config) { {obsolete_setting[:name] => obsolete_setting[:sample_value]} }
+          it "should raise a config error with the appropriate message" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /The setting `#{obsolete_setting[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. Set '#{obsolete_setting[:replacement]}' instead/i
+          end
+        end
+      end
+    end
+
     context "against not authentic Elasticsearch" do
       before(:each) do
          Elasticsearch::Client.send(:define_method, :ping) { raise Elasticsearch::UnsupportedProductError.new("Fake error") } # define error ping method
@@ -86,11 +92,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
         before do
           allow(Elasticsearch::Client).to receive(:new).and_return(es_client)
-          if elastic_ruby_v8_client_available?
-            allow(es_client).to receive(:info).and_raise(Elastic::Transport::Transport::Errors::BadRequest.new)
-          else
-            allow(es_client).to receive(:info).and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest.new)
-          end
+          allow(es_client).to receive(:info).and_raise(
+            Elasticsearch::Transport::Transport::Errors::BadRequest.new
+          )
         end
 
         it "raises an exception" do
@@ -662,28 +666,11 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         context 'if the `docinfo_target` exist but is not of type hash' do
           let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
           let(:do_register) { false }
-          let(:mock_queue) { double('Queue', :<< => nil) }
-          let(:hit) { response.dig('hits', 'hits').first }
-
-          it 'emits a tagged event with JSON-serialized event in [event][original]' do
-            allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
 
+          it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
+            expect(client).not_to receive(:clear_scroll)
             plugin.register
-            plugin.run(mock_queue)
-
-            expect(mock_queue).to have_received(:<<) do |event|
-              expect(event).to be_a_kind_of LogStash::Event
-
-              expect(event.get('tags')).to include("_elasticsearch_input_failure")
-              expect(event.get('[event][original]')).to be_a_kind_of String
-              expect(JSON.load(event.get('[event][original]'))).to eq hit
-            end
-
-            expect(plugin.logger)
-              .to have_received(:warn).with(
-                a_string_including("Event creation error, original data now in [event][original] field"),
-                a_hash_including(:message => a_string_including('unable to merge docinfo fields into docinfo_target=`metadata_with_string`'),
-                                 :data    => a_string_including('"_id":"C5b2xLQwTZa76jBmHIbwHQ"')))
+            expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
           end
 
         end
@@ -740,13 +727,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:client)
-        target_field = :@seeds
-        begin
-          Elasticsearch::Transport::Client
-        rescue
-          target_field = :@hosts
-        end
-        expect( client.transport.instance_variable_get(target_field) ).to eql [{
+
+        expect( client.transport.instance_variable_get(:@seeds) ).to eql [{
                                                                               :scheme => "https",
                                                                               :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
                                                                               :port => 9243,
@@ -1152,7 +1134,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
     context "when there's an exception" do
       before(:each) do
-        allow(client).to receive(:search).and_raise RuntimeError.new("test exception")
+        allow(client).to receive(:search).and_raise RuntimeError
       end
       it 'produces no events' do
         plugin.run queue
@@ -1266,220 +1248,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
   end
 
-  context '#push_hit' do
-    let(:config) do
-      {
-        'docinfo' => true, # include ids
-        'docinfo_target' => '[@metadata][docinfo]'
-      }
-    end
-
-    let(:hit) do
-      JSON.load(<<~EOJSON)
-        {
-          "_index" : "test_bulk_index_2",
-          "_type" : "_doc",
-          "_id" : "sHe6A3wBesqF7ydicQvG",
-          "_score" : 1.0,
-          "_source" : {
-            "@timestamp" : "2021-09-20T15:02:02.557Z",
-            "message" : "ping",
-            "@version" : "17",
-            "sequence" : 7,
-            "host" : {
-              "name" : "maybe.local",
-              "ip" : "127.0.0.1"
-            }
-          }
-        }
-      EOJSON
-    end
-
-    let(:mock_queue) { double('queue', :<< => nil) }
-
-    before(:each) do
-      plugin.send(:setup_cursor_tracker)
-    end
-
-    it 'pushes a generated event to the queue' do
-      plugin.send(:push_hit, hit, mock_queue)
-      expect(mock_queue).to have_received(:<<) do |event|
-        expect(event).to be_a_kind_of LogStash::Event
-
-        # fields overriding defaults
-        expect(event.timestamp.to_s).to eq("2021-09-20T15:02:02.557Z")
-        expect(event.get('@version')).to eq("17")
-
-        # structure from hit's _source
-        expect(event.get('message')).to eq("ping")
-        expect(event.get('sequence')).to eq(7)
-        expect(event.get('[host][name]')).to eq("maybe.local")
-        expect(event.get('[host][ip]')).to eq("127.0.0.1")
-
-        # docinfo fields
-        expect(event.get('[@metadata][docinfo][_index]')).to eq("test_bulk_index_2")
-        expect(event.get('[@metadata][docinfo][_type]')).to eq("_doc")
-        expect(event.get('[@metadata][docinfo][_id]')).to eq("sHe6A3wBesqF7ydicQvG")
-      end
-    end
-
-    context 'when event creation fails' do
-      before(:each) do
-        allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
-
-        allow(plugin.event_factory).to receive(:new_event).and_call_original
-        allow(plugin.event_factory).to receive(:new_event).with(a_hash_including hit['_source']).and_raise(RuntimeError, 'intentional')
-      end
-
-      it 'pushes a tagged event containing a JSON-encoded hit in [event][original]' do
-        plugin.send(:push_hit, hit, mock_queue)
-
-        expect(mock_queue).to have_received(:<<) do |event|
-          expect(event).to be_a_kind_of LogStash::Event
-
-          expect(event.get('tags')).to include("_elasticsearch_input_failure")
-          expect(event.get('[event][original]')).to be_a_kind_of String
-          expect(JSON.load(event.get('[event][original]'))).to eq hit
-        end
-
-        expect(plugin.logger)
-          .to have_received(:warn).with(
-            a_string_including("Event creation error, original data now in [event][original] field"),
-            a_hash_including(:message => a_string_including('intentional'),
-                             :data    => a_string_including('"_id":"sHe6A3wBesqF7ydicQvG"')))
-
-      end
-    end
-  end
-
   # @note can be removed once we depends on elasticsearch gem >= 6.x
   def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
-  describe "#ESQL" do
-    let(:config) do
-      {
-        "query" => "FROM test-index | STATS count() BY field",
-        "query_type" => "esql",
-        "retries" => 3
-      }
-    end
-    let(:es_version) { LogStash::Inputs::Elasticsearch::ES_ESQL_SUPPORT_VERSION }
-    let(:ls_version) { LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION }
-
-    before(:each) do
-      stub_const("LOGSTASH_VERSION", ls_version)
-    end
-
-    describe "#initialize" do
-      it "sets up the ESQL client with correct parameters" do
-        expect(plugin.instance_variable_get(:@query_type)).to eq(config["query_type"])
-        expect(plugin.instance_variable_get(:@query)).to eq(config["query"])
-        expect(plugin.instance_variable_get(:@retries)).to eq(config["retries"])
-      end
-    end
-
-    describe "#register" do
-      before(:each) do
-        Elasticsearch::Client.send(:define_method, :ping) { }
-        allow_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(cluster_info)
-      end
-      it "creates ES|QL executor" do
-        plugin.register
-        expect(plugin.instance_variable_get(:@query_executor)).to be_an_instance_of(LogStash::Inputs::Elasticsearch::Esql)
-      end
-    end
-
-    describe "#validation" do
-
-      describe "LS version" do
-        context "when compatible" do
-
-          it "does not raise an error" do
-            expect { plugin.send(:validate_ls_version_for_esql_support!) }.not_to raise_error
-          end
-        end
-
-        context "when incompatible" do
-          before(:each) do
-            stub_const("LOGSTASH_VERSION", "8.10.0")
-          end
-
-          it "raises a runtime error" do
-            expect { plugin.send(:validate_ls_version_for_esql_support!) }
-              .to raise_error(RuntimeError, /Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{ls_version}/)
-          end
-        end
-      end
-
-      describe "ES version" do
-        before(:each) do
-          allow(plugin).to receive(:es_version).and_return("8.10.5")
-        end
-
-        context "when incompatible" do
-          it "raises a runtime error" do
-            expect { plugin.send(:validate_es_for_esql_support!) }
-              .to raise_error(RuntimeError, /Connected Elasticsearch 8.10.5 version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{es_version} version./)
-          end
-        end
-      end
-
-      context "ES|QL query and DSL params used together" do
-        let(:config) {
-          super().merge({
-            "index" => "my-index",
-            "size" => 1,
-            "slices" => 1,
-            "search_api" => "auto",
-            "docinfo" => true,
-            "docinfo_target" => "[@metadata][docinfo]",
-            "docinfo_fields" => ["_index"],
-            "response_type" => "hits",
-            "tracking_field" => "[@metadata][tracking]"
-          })}
-
-        it "raises a config error" do
-          mixed_fields = %w[index size slices docinfo_fields response_type tracking_field]
-          expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Configured #{mixed_fields} params are not allowed while using ES|QL query/)
-        end
-      end
-
-      describe "ES|QL query"  do
-        context "when query is valid" do
-          it "does not raise an error" do
-            expect { plugin.send(:validate_esql_query!) }.not_to raise_error
-          end
-        end
-
-        context "when query is empty" do
-          let(:config) do
-            {
-              "query" => " "
-            }
-          end
-
-          it "raises a configuration error" do
-            expect { plugin.send(:validate_esql_query!) }
-              .to raise_error(LogStash::ConfigurationError, /`query` cannot be empty/)
-          end
-        end
-
-        context "when query doesn't align with ES syntax" do
-          let(:config) do
-            {
-              "query" => "RANDOM query"
-            }
-          end
-
-          it "raises a configuration error" do
-            source_commands = %w[FROM ROW SHOW]
-            expect { plugin.send(:validate_esql_query!) }
-              .to raise_error(LogStash::ConfigurationError, "`query` needs to start with any of #{source_commands}")
-          end
-        end
-      end
-    end
-  end
 end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -4,7 +4,7 @@ require "logstash/plugin"
 require "logstash/inputs/elasticsearch"
 require_relative "../../../spec/es_helper"
 
-describe LogStash::Inputs::Elasticsearch do
+describe LogStash::Inputs::Elasticsearch, :integration => true do
 
   SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
 
@@ -76,14 +76,6 @@ describe LogStash::Inputs::Elasticsearch do
     shared_examples 'secured_elasticsearch' do
       it_behaves_like 'an elasticsearch index plugin'
 
-      let(:unauth_exception_class) do
-        begin
-          Elasticsearch::Transport::Transport::Errors::Unauthorized
-        rescue
-          Elastic::Transport::Transport::Errors::Unauthorized
-        end
-      end
-
       context "incorrect auth credentials" do
 
         let(:config) do
@@ -93,7 +85,7 @@ describe LogStash::Inputs::Elasticsearch do
         let(:queue) { [] }
 
         it "fails to run the plugin" do
-          expect { plugin.register }.to raise_error unauth_exception_class
+          expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.23.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-06 00:00:00.000000000 Z
+date: 2024-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -92,9 +92,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.17.9
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '9'
   name: elasticsearch
   type: :runtime
   prerelease: false
@@ -103,9 +100,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.17.9
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '9'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -278,29 +272,21 @@ files:
 - lib/logstash/helpers/loggable_try.rb
 - lib/logstash/inputs/elasticsearch.rb
 - lib/logstash/inputs/elasticsearch/aggregation.rb
-- lib/logstash/inputs/elasticsearch/cursor_tracker.rb
-- lib/logstash/inputs/elasticsearch/esql.rb
 - lib/logstash/inputs/elasticsearch/paginated_search.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
-- spec/fixtures/test_certs/GENERATED_AT
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
-- spec/fixtures/test_certs/es.chain.crt
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
-- spec/fixtures/test_certs/renew.sh
-- spec/inputs/cursor_tracker_spec.rb
-- spec/inputs/elasticsearch_esql_spec.rb
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
-- spec/inputs/integration/elasticsearch_esql_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 - spec/inputs/paginated_search_spec.rb
-homepage: https://elastic.co/logstash
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
 metadata:
@@ -327,18 +313,12 @@ specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:
 - spec/es_helper.rb
-- spec/fixtures/test_certs/GENERATED_AT
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
-- spec/fixtures/test_certs/es.chain.crt
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
-- spec/fixtures/test_certs/renew.sh
-- spec/inputs/cursor_tracker_spec.rb
-- spec/inputs/elasticsearch_esql_spec.rb
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
-- spec/inputs/integration/elasticsearch_esql_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 - spec/inputs/paginated_search_spec.rb

data/lib/logstash/inputs/elasticsearch/cursor_tracker.rb

@@ -1,58 +0,0 @@
-require 'fileutils'
-
-module LogStash; module Inputs; class Elasticsearch
-  class CursorTracker
-    include LogStash::Util::Loggable
-
-    attr_reader :last_value
-
-    def initialize(last_run_metadata_path:, tracking_field:, tracking_field_seed:)
-      @last_run_metadata_path = last_run_metadata_path
-      @last_value_hashmap = Java::java.util.concurrent.ConcurrentHashMap.new
-      @last_value = IO.read(@last_run_metadata_path) rescue nil || tracking_field_seed
-      @tracking_field = tracking_field
-      logger.info "Starting value for cursor field \"#{@tracking_field}\": #{@last_value}"
-      @mutex = Mutex.new
-    end
-
-    def checkpoint_cursor(intermediate: true)
-      @mutex.synchronize do
-        if intermediate
-          # in intermediate checkpoints pick the smallest
-          converge_last_value {|v1, v2| v1 < v2 ? v1 : v2}
-        else
-          # in the last search of a PIT choose the largest
-          converge_last_value {|v1, v2| v1 > v2 ? v1 : v2}
-          @last_value_hashmap.clear
-        end
-        IO.write(@last_run_metadata_path, @last_value)
-      end
-    end
-
-    def converge_last_value(&block)
-      return if @last_value_hashmap.empty?
-      new_last_value = @last_value_hashmap.reduceValues(1000, &block)
-      logger.debug? && logger.debug("converge_last_value: got #{@last_value_hashmap.values.inspect}. won: #{new_last_value}")
-      return if new_last_value == @last_value
-      @last_value = new_last_value
-      logger.info "New cursor value for field \"#{@tracking_field}\" is: #{new_last_value}"
-    end
-
-    def record_last_value(event)
-      value = event.get(@tracking_field)
-      logger.trace? && logger.trace("storing last_value if #{@tracking_field} for #{Thread.current.object_id}: #{value}")
-      @last_value_hashmap.put(Thread.current.object_id, value)
-    end
-
-    def inject_cursor(query_json)
-      # ":present" means "now - 30s" to avoid grabbing partially visible data in the PIT
-      result = query_json.gsub(":last_value", @last_value.to_s).gsub(":present", now_minus_30s)
-      logger.debug("inject_cursor: injected values for ':last_value' and ':present'", :query => result)
-      result
-    end
-
-    def now_minus_30s
-      Java::java.time.Instant.now.minusSeconds(30).to_s
-    end
-  end
-end; end; end

data/lib/logstash/inputs/elasticsearch/esql.rb

@@ -1,153 +0,0 @@
-require 'logstash/helpers/loggable_try'
-
-module LogStash
-  module Inputs
-    class Elasticsearch
-      class Esql
-        include LogStash::Util::Loggable
-
-        ESQL_JOB = "ES|QL job"
-
-        ESQL_PARSERS_BY_TYPE = Hash.new(lambda { |x| x }).merge(
-          'date' => ->(value) { value && LogStash::Timestamp.new(value) },
-          )
-
-        # Initialize the ESQL query executor
-        # @param client [Elasticsearch::Client] The Elasticsearch client instance
-        # @param plugin [LogStash::Inputs::Elasticsearch] The parent plugin instance
-        def initialize(client, plugin)
-          @client = client
-          @event_decorator = plugin.method(:decorate_event)
-          @retries = plugin.params["retries"]
-
-          target_field = plugin.params["target"]
-          if target_field
-            def self.apply_target(path); "[#{target_field}][#{path}]"; end
-          else
-            def self.apply_target(path); path; end
-          end
-
-          @query = plugin.params["query"]
-          unless @query.include?('METADATA')
-            logger.info("`METADATA` not found the query. `_id`, `_version` and `_index` will not be available in the result", {:query => @query})
-          end
-          logger.debug("ES|QL executor initialized with", {:query => @query})
-        end
-
-        # Execute the ESQL query and process results
-        # @param output_queue [Queue] The queue to push processed events to
-        # @param query A query (to obey interface definition)
-        def do_run(output_queue, query)
-          logger.info("ES|QL executor has started")
-          response = retryable(ESQL_JOB) do
-            @client.esql.query({ body: { query: @query }, format: 'json', drop_null_columns: true })
-          end
-          # retriable already printed error details
-          return if response == false
-
-          if response&.headers&.dig("warning")
-            logger.warn("ES|QL executor received warning", {:warning_message => response.headers["warning"]})
-          end
-          columns = response['columns']&.freeze
-          values = response['values']&.freeze
-          logger.debug("ES|QL query response size: #{values&.size}")
-
-          process_response(columns, values, output_queue) if columns && values
-        end
-
-        # Execute a retryable operation with proper error handling
-        # @param job_name [String] Name of the job for logging purposes
-        # @yield The block to execute
-        # @return [Boolean] true if successful, false otherwise
-        def retryable(job_name, &block)
-          stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
-          stud_try.try((@retries + 1).times) { yield }
-        rescue => e
-          error_details = {:message => e.message, :cause => e.cause}
-          error_details[:backtrace] = e.backtrace if logger.debug?
-          logger.error("#{job_name} failed with ", error_details)
-          false
-        end
-
-        private
-
-        # Process the ESQL response and push events to the output queue
-        # @param columns [Array[Hash]] The ESQL query response columns
-        # @param values [Array[Array]] The ESQL query response hits
-        # @param output_queue [Queue] The queue to push processed events to
-        def process_response(columns, values, output_queue)
-          column_specs = columns.map { |column| ColumnSpec.new(column) }
-          sub_element_mark_map = mark_sub_elements(column_specs)
-          multi_fields = sub_element_mark_map.filter_map { |key, val| key.name if val == true }
-          logger.warn("Multi-fields found in ES|QL result and they will not be available in the event. Please use `RENAME` command if you want to include them.", { :detected_multi_fields => multi_fields }) if multi_fields.any?
-
-          values.each do |row|
-            event = column_specs.zip(row).each_with_object(LogStash::Event.new) do |(column, value), event|
-              # `unless value.nil?` is a part of `drop_null_columns` that if some of columns' values are not `nil`, `nil` values appear
-              # we should continuously filter out them to achieve full `drop_null_columns` on each individual row (ideal `LIMIT 1` result)
-              # we also exclude sub-elements of main field
-              if value && sub_element_mark_map[column] == false
-                field_reference = apply_target(column.field_reference)
-                event.set(field_reference, ESQL_PARSERS_BY_TYPE[column.type].call(value))
-              end
-            end
-            @event_decorator.call(event)
-            output_queue << event
-          rescue => e
-            # if event creation fails with whatever reason, inform user and tag with failure and return entry as it is
-            logger.warn("Event creation error, ", message: e.message, exception: e.class, data: { "columns" => columns, "values" => [row] })
-            failed_event = LogStash::Event.new("columns" => columns, "values" => [row], "tags" => ['_elasticsearch_input_failure'])
-            output_queue << failed_event
-          end
-        end
-
-        # Determines whether each column in a collection is a nested sub-element (example "user.age")
-        # of another column in the same collection (example "user").
-        #
-        # @param columns [Array<ColumnSpec>] An array of objects with a `name` attribute representing field paths.
-        # @return [Hash<ColumnSpec, Boolean>] A hash mapping each column to `true` if it is a sub-element of another field, `false` otherwise.
-        # Time complexity: (O(NlogN+N*K)) where K is the number of conflict depth
-        #   without (`prefix_set`) memoization, it would be O(N^2)
-        def mark_sub_elements(columns)
-          # Sort columns by name length (ascending)
-          sorted_columns = columns.sort_by { |c| c.name.length }
-          prefix_set = Set.new # memoization set
-
-          sorted_columns.each_with_object({}) do |column, memo|
-            # Split the column name into parts (e.g., "user.profile.age" → ["user", "profile", "age"])
-            parts = column.name.split('.')
-
-            # Generate all possible parent prefixes (e.g., "user", "user.profile")
-            # and check if any parent prefix exists in the set
-            parent_prefixes = (0...parts.size - 1).map { |i| parts[0..i].join('.') }
-            memo[column] = parent_prefixes.any? { |prefix| prefix_set.include?(prefix) }
-            prefix_set.add(column.name)
-          end
-        end
-      end
-
-      # Class representing a column specification in the ESQL response['columns']
-      # The class's main purpose is to provide a structure for the event key
-      # columns is an array with `name` and `type` pair (example: `{"name"=>"@timestamp", "type"=>"date"}`)
-      # @attr_reader :name [String] The name of the column
-      # @attr_reader :type [String] The type of the column
-      class ColumnSpec
-        attr_reader :name, :type
-
-        def initialize(spec)
-          @name = isolate(spec.fetch('name'))
-          @type = isolate(spec.fetch('type'))
-        end
-
-        def field_reference
-          @_field_reference ||= '[' + name.gsub('.', '][') + ']'
-        end
-
-        private
-        def isolate(value)
-          value.frozen? ? value : value.clone.freeze
-        end
-      end
-    end
-  end
-end

data/spec/fixtures/test_certs/GENERATED_AT

@@ -1 +0,0 @@
-2024-12-26T22:27:15+00:00