logstash-input-elasticsearch 4.21.2 → 4.23.0

data/lib/logstash/inputs/elasticsearch/paginated_search.rb

@@ -21,9 +21,10 @@ module LogStash
           @pipeline_id = plugin.pipeline_id
         end
 
-        def do_run(output_queue)
-          return retryable_search(output_queue) if @slices.nil? || @slices <= 1
+        def do_run(output_queue, query)
+          @query = query
 
+          return retryable_search(output_queue) if @slices.nil? || @slices <= 1
           retryable_slice_search(output_queue)
         end
 
@@ -122,6 +123,13 @@ module LogStash
         PIT_JOB = "create point in time (PIT)"
         SEARCH_AFTER_JOB = "search_after paginated search"
 
+        attr_accessor :cursor_tracker
+
+        def do_run(output_queue, query)
+          super(output_queue, query)
+          @cursor_tracker.checkpoint_cursor(intermediate: false) if @cursor_tracker
+        end
+
         def pit?(id)
           !!id&.is_a?(String)
         end
@@ -192,6 +200,8 @@ module LogStash
             end
           end
 
+          @cursor_tracker.checkpoint_cursor(intermediate: true) if @cursor_tracker
+
           logger.info("Query completed", log_details)
         end
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -73,6 +73,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   require 'logstash/inputs/elasticsearch/paginated_search'
   require 'logstash/inputs/elasticsearch/aggregation'
+  require 'logstash/inputs/elasticsearch/cursor_tracker'
+  require 'logstash/inputs/elasticsearch/esql'
 
   include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
   include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
@@ -95,15 +97,21 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # The index or alias to search.
   config :index, :validate => :string, :default => "logstash-*"
 
-  # The query to be executed. Read the Elasticsearch query DSL documentation
-  # for more info
-  # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+  # A type of Elasticsearch query, provided by @query. This will validate query shape and other params.
+  config :query_type, :validate => %w[dsl esql], :default => 'dsl'
+
+  # The query to be executed. DSL or ES|QL (when `query_type => 'esql'`) query shape is accepted.
+  # Read the following documentations for more info
+  # Query DSL: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+  # ES|QL: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html
   config :query, :validate => :string, :default => '{ "sort": [ "_doc" ] }'
 
-  # This allows you to speccify the response type: either hits or aggregations
-  # where hits: normal search request
-  #       aggregations: aggregation request
-  config :response_type, :validate => ['hits', 'aggregations'], :default => 'hits'
+  # This allows you to specify the DSL response type: one of [hits, aggregations]
+  # where
+  #   hits: normal search request
+  #   aggregations: aggregation request
+  # Note that this param is invalid when `query_type => 'esql'`, ES|QL response shape is always a tabular format
+  config :response_type, :validate => %w[hits aggregations], :default => 'hits'
 
   # This allows you to set the maximum number of hits returned per scroll.
   config :size, :validate => :number, :default => 1000
@@ -124,6 +132,20 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # by this pipeline input.
   config :slices, :validate => :number
 
+  # Enable tracking the value of a given field to be used as a cursor
+  # Main concerns:
+  #       * using anything other than _event.timestamp easily leads to data loss
+  #       * the first "synchronization run can take a long time"
+  config :tracking_field, :validate => :string
+
+  # Define the initial seed value of the tracking_field
+  config :tracking_field_seed, :validate => :string, :default => "1970-01-01T00:00:00.000000000Z"
+
+  # The location of where the tracking field value will be stored
+  # The value is persisted after each scheduled run (and not per result)
+  # If it's not set it defaults to '${path.data}/plugins/inputs/elasticsearch/<pipeline_id>/last_run_value'
+  config :last_run_metadata_path, :validate => :string
+
   # If set, include Elasticsearch document information such as index, type, and
   # the id in the event.
   #
@@ -262,6 +284,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # exactly once.
   config :schedule, :validate => :string
 
+  # Allow scheduled runs to overlap (enabled by default). Setting to false will
+  # only start a new scheduled run after the previous one completes.
+  config :schedule_overlap, :validate => :boolean
+
   # If set, the _source of each hit will be added nested under the target instead of at the top-level
   config :target, :validate => :field_reference
 
@@ -274,6 +300,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
   INTERNAL_ORIGIN_HEADER = { 'x-elastic-product-origin' => 'logstash-input-elasticsearch'}.freeze
 
+  LS_ESQL_SUPPORT_VERSION = "8.17.4" # the version started using elasticsearch-ruby v8
+  ES_ESQL_SUPPORT_VERSION = "8.11.0"
+
   def initialize(params={})
     super(params)
 
@@ -290,10 +319,17 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     fill_hosts_from_cloud_id
     setup_ssl_params!
 
-    @base_query = LogStash::Json.load(@query)
-    if @slices
-      @base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
-      @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
+    if @query_type == 'esql'
+      validate_ls_version_for_esql_support!
+      validate_esql_query!
+      not_allowed_options = original_params.keys & %w(index size slices search_api docinfo docinfo_target docinfo_fields response_type tracking_field)
+      raise(LogStash::ConfigurationError, "Configured #{not_allowed_options} params are not allowed while using ES|QL query") if not_allowed_options&.size > 1
+    else
+      @base_query = LogStash::Json.load(@query)
+      if @slices
+        @base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
+        @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
+      end
     end
 
     @retries < 0 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `retries` option must be equal or greater than zero, got `#{@retries}`")
@@ -329,21 +365,27 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     test_connection!
 
+    validate_es_for_esql_support!
+
     setup_serverless
 
     setup_search_api
 
-    setup_query_executor
+    @query_executor = create_query_executor
+
+    setup_cursor_tracker
 
     @client
   end
 
   def run(output_queue)
     if @schedule
-      scheduler.cron(@schedule) { @query_executor.do_run(output_queue) }
+      scheduler.cron(@schedule, :overlap => @schedule_overlap) do
+        @query_executor.do_run(output_queue, get_query_object())
+      end
       scheduler.join
     else
-      @query_executor.do_run(output_queue)
+      @query_executor.do_run(output_queue, get_query_object())
     end
   end
 
@@ -354,6 +396,28 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     event = event_from_hit(hit, root_field)
     decorate(event)
     output_queue << event
+    record_last_value(event)
+  end
+
+  def decorate_event(event)
+    decorate(event)
+  end
+
+  private
+
+  def get_query_object
+    return @query if @query_type == 'esql'
+    if @cursor_tracker
+      query = @cursor_tracker.inject_cursor(@query)
+      @logger.debug("new query is #{query}")
+    else
+      query = @query
+    end
+    LogStash::Json.load(query)
+  end
+
+  def record_last_value(event)
+    @cursor_tracker.record_last_value(event) if @tracking_field
   end
 
   def event_from_hit(hit, root_field)
@@ -383,8 +447,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     event.set(@docinfo_target, docinfo_target)
   end
 
-  private
-
   def hosts_default?(hosts)
     hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
   end
@@ -662,18 +724,38 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   end
 
-  def setup_query_executor
-    @query_executor = case @response_type
-                      when 'hits'
-                        if @resolved_search_api == "search_after"
-                          LogStash::Inputs::Elasticsearch::SearchAfter.new(@client, self)
-                        else
-                          logger.warn("scroll API is no longer recommended for pagination. Consider using search_after instead.") if es_major_version >= 8
-                          LogStash::Inputs::Elasticsearch::Scroll.new(@client, self)
-                        end
-                      when 'aggregations'
-                        LogStash::Inputs::Elasticsearch::Aggregation.new(@client, self)
-                      end
+  def create_query_executor
+    return LogStash::Inputs::Elasticsearch::Esql.new(@client, self) if @query_type == 'esql'
+
+    # DSL query executor
+    return LogStash::Inputs::Elasticsearch::Aggregation.new(@client, self) if @response_type == 'aggregations'
+    # response_type is hits, executor can be search_after or scroll type
+    return LogStash::Inputs::Elasticsearch::SearchAfter.new(@client, self) if @resolved_search_api == "search_after"
+
+    logger.warn("scroll API is no longer recommended for pagination. Consider using search_after instead.") if es_major_version >= 8
+    LogStash::Inputs::Elasticsearch::Scroll.new(@client, self)
+  end
+
+  def setup_cursor_tracker
+    return unless @tracking_field
+    return unless @query_executor.is_a?(LogStash::Inputs::Elasticsearch::SearchAfter)
+
+    if @resolved_search_api != "search_after" || @response_type != "hits"
+      raise ConfigurationError.new("The `tracking_field` feature can only be used with `search_after` non-aggregation queries")
+    end
+
+    @cursor_tracker = CursorTracker.new(last_run_metadata_path: last_run_metadata_path,
+                                        tracking_field: @tracking_field,
+                                        tracking_field_seed: @tracking_field_seed)
+    @query_executor.cursor_tracker = @cursor_tracker
+  end
+
+  def last_run_metadata_path
+    return @last_run_metadata_path if @last_run_metadata_path
+
+    last_run_metadata_path = ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "elasticsearch", pipeline_id, "last_run_value")
+    FileUtils.mkdir_p ::File.dirname(last_run_metadata_path)
+    last_run_metadata_path
   end
 
   def get_transport_client_class
@@ -690,6 +772,26 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     ::Elastic::Transport::Transport::HTTP::Manticore
   end
 
+  def validate_ls_version_for_esql_support!
+    if Gem::Version.create(LOGSTASH_VERSION) < Gem::Version.create(LS_ESQL_SUPPORT_VERSION)
+      fail("Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{LS_ESQL_SUPPORT_VERSION}")
+    end
+  end
+
+  def validate_esql_query!
+    fail(LogStash::ConfigurationError, "`query` cannot be empty") if @query.strip.empty?
+    source_commands = %w[FROM ROW SHOW]
+    contains_source_command = source_commands.any? { |source_command| @query.strip.start_with?(source_command) }
+    fail(LogStash::ConfigurationError, "`query` needs to start with any of #{source_commands}") unless contains_source_command
+  end
+
+  def validate_es_for_esql_support!
+    return unless @query_type == 'esql'
+    # make sure connected ES supports ES|QL (8.11+)
+    es_supports_esql = Gem::Version.create(es_version) >= Gem::Version.create(ES_ESQL_SUPPORT_VERSION)
+    fail("Connected Elasticsearch #{es_version} version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{ES_ESQL_SUPPORT_VERSION} version.") unless es_supports_esql
+  end
+
   module URIOrEmptyValidator
     ##
     # @override to provide :uri_or_empty validator

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.21.2'
+  s.version         = '4.23.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/cursor_tracker_spec.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/devutils/rspec/shared_examples"
+require "logstash/inputs/elasticsearch"
+require "logstash/inputs/elasticsearch/cursor_tracker"
+
+describe LogStash::Inputs::Elasticsearch::CursorTracker do
+
+  let(:last_run_metadata_path) { Tempfile.new('cursor_tracker_testing').path }
+  let(:tracking_field_seed) { "1980-01-01T23:59:59.999999999Z" }
+  let(:options) do
+    {
+      :last_run_metadata_path => last_run_metadata_path,
+      :tracking_field => "my_field",
+      :tracking_field_seed => tracking_field_seed
+    }
+  end
+
+  subject { described_class.new(**options) }
+
+  it "creating a class works" do
+    expect(subject).to be_a described_class
+  end
+
+  describe "checkpoint_cursor" do
+    before(:each) do
+      subject.checkpoint_cursor(intermediate: false) # store seed value
+      [
+        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-03T23:59:59.999999999Z")) },
+        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-01T23:59:59.999999999Z")) },
+        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-02T23:59:59.999999999Z")) },
+      ].each(&:join)
+    end
+    context "when doing intermediate checkpoint" do
+      it "persists the smallest value" do
+        subject.checkpoint_cursor(intermediate: true)
+        expect(IO.read(last_run_metadata_path)).to eq("2025-01-01T23:59:59.999999999Z")
+      end
+    end
+    context "when doing non-intermediate checkpoint" do
+      it "persists the largest value" do
+        subject.checkpoint_cursor(intermediate: false)
+        expect(IO.read(last_run_metadata_path)).to eq("2025-01-03T23:59:59.999999999Z")
+      end
+    end
+  end
+
+  describe "inject_cursor" do
+    let(:new_value) { "2025-01-03T23:59:59.999999999Z" }
+    let(:fake_now) { "2026-09-19T23:59:59.999999999Z" }
+
+    let(:query) do
+      %q[
+      { "query": { "range": { "event.ingested": { "gt": :last_value, "lt": :present}}}, "sort": [ { "event.ingested": {"order": "asc", "format": "strict_date_optional_time_nanos", "numeric_type" : "date_nanos" } } ] }
+      ]
+    end
+
+    before(:each) do
+      subject.record_last_value(LogStash::Event.new("my_field" => new_value))
+      subject.checkpoint_cursor(intermediate: false)
+      allow(subject).to receive(:now_minus_30s).and_return(fake_now)
+    end
+
+    it "injects the value of the cursor into json query if it contains :last_value" do
+      expect(subject.inject_cursor(query)).to match(/#{new_value}/)
+    end
+
+    it "injects current time into json query if it contains :present" do
+      expect(subject.inject_cursor(query)).to match(/#{fake_now}/)
+    end
+  end
+end

data/spec/inputs/elasticsearch_esql_spec.rb

@@ -0,0 +1,180 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/elasticsearch"
+require "elasticsearch"
+
+describe LogStash::Inputs::Elasticsearch::Esql do
+  let(:client) { instance_double(Elasticsearch::Client) }
+  let(:esql_client) { double("esql-client") }
+
+  let(:plugin) { instance_double(LogStash::Inputs::Elasticsearch, params: plugin_config, decorate_event: nil) }
+  let(:plugin_config) do
+    {
+      "query" => "FROM test-index | STATS count() BY field",
+      "retries" => 3
+    }
+  end
+  let(:esql_executor) { described_class.new(client, plugin) }
+
+  describe "#initialization" do
+    it "sets up the ESQL client with correct parameters" do
+      expect(esql_executor.instance_variable_get(:@query)).to eq(plugin_config["query"])
+      expect(esql_executor.instance_variable_get(:@retries)).to eq(plugin_config["retries"])
+      expect(esql_executor.instance_variable_get(:@target_field)).to eq(nil)
+    end
+  end
+
+  describe "#execution" do
+    let(:output_queue) { Queue.new }
+
+    context "when faces error while retrying" do
+      it "retries the given block the specified number of times" do
+        attempts = 0
+        result = esql_executor.retryable("Test Job") do
+          attempts += 1
+          raise StandardError if attempts < 3
+          "success"
+        end
+        expect(attempts).to eq(3)
+        expect(result).to eq("success")
+      end
+
+      it "returns false if the block fails all attempts" do
+        result = esql_executor.retryable("Test Job") do
+          raise StandardError
+        end
+        expect(result).to eq(false)
+      end
+    end
+
+    context "when executing chain of processes" do
+      let(:response) { { 'values' => [%w[foo bar]], 'columns' => [{ 'name' => 'a.b.1.d', 'type' => 'keyword' },
+                                                                  { 'name' => 'h_g.k$l.m.0', 'type' => 'keyword' }] } }
+
+      before do
+        allow(esql_executor).to receive(:retryable).and_yield
+        allow(client).to receive_message_chain(:esql, :query).and_return(response)
+      end
+
+      it "executes the ESQL query and processes the results" do
+        allow(response).to receive(:headers).and_return({})
+        esql_executor.do_run(output_queue, plugin_config["query"])
+        expect(output_queue.size).to eq(1)
+
+        event = output_queue.pop
+        expect(event.get('[a][b][1][d]')).to eq('foo')
+        expect(event.get('[h_g][k$l][m][0]')).to eq('bar')
+      end
+
+      it "logs a warning if the response contains a warning header" do
+        allow(response).to receive(:headers).and_return({ "warning" => "some warning" })
+        expect(esql_executor.logger).to receive(:warn).with("ES|QL executor received warning", { :warning_message => "some warning" })
+        esql_executor.do_run(output_queue, plugin_config["query"])
+      end
+
+      it "does not log a warning if the response does not contain a warning header" do
+        allow(response).to receive(:headers).and_return({})
+        expect(esql_executor.logger).not_to receive(:warn)
+        esql_executor.do_run(output_queue, plugin_config["query"])
+      end
+    end
+
+    describe "multiple rows in the result" do
+      let(:response) { { 'values' => rows, 'columns' => [{ 'name' => 'key.1', 'type' => 'keyword' },
+                                                         { 'name' => 'key.2', 'type' => 'keyword' }] } }
+
+      before do
+        allow(esql_executor).to receive(:retryable).and_yield
+        allow(client).to receive_message_chain(:esql, :query).and_return(response)
+        allow(response).to receive(:headers).and_return({})
+      end
+
+      context "when mapping" do
+        let(:rows) { [%w[foo bar], %w[hello world]] }
+
+        it "1:1 maps rows to events" do
+          esql_executor.do_run(output_queue, plugin_config["query"])
+          expect(output_queue.size).to eq(2)
+
+          event_1 = output_queue.pop
+          expect(event_1.get('[key][1]')).to eq('foo')
+          expect(event_1.get('[key][2]')).to eq('bar')
+
+          event_2 = output_queue.pop
+          expect(event_2.get('[key][1]')).to eq('hello')
+          expect(event_2.get('[key][2]')).to eq('world')
+        end
+      end
+
+      context "when partial nil values appear" do
+        let(:rows) { [[nil, "bar"], ["hello", nil]] }
+
+        it "ignores the nil values" do
+          esql_executor.do_run(output_queue, plugin_config["query"])
+          expect(output_queue.size).to eq(2)
+
+          event_1 = output_queue.pop
+          expect(event_1.get('[key][1]')).to eq(nil)
+          expect(event_1.get('[key][2]')).to eq('bar')
+
+          event_2 = output_queue.pop
+          expect(event_2.get('[key][1]')).to eq('hello')
+          expect(event_2.get('[key][2]')).to eq(nil)
+        end
+      end
+    end
+
+    context "when sub-elements occur in the result" do
+      let(:response) { {
+        'values' => [[50, 1, 100], [50, 0, 1000], [50, 9, 99999]],
+        'columns' =>
+          [
+            { 'name' => 'time', 'type' => 'long' },
+            { 'name' => 'time.min', 'type' => 'long' },
+            { 'name' => 'time.max', 'type' => 'long' },
+          ]
+      } }
+
+      before do
+        allow(esql_executor).to receive(:retryable).and_yield
+        allow(client).to receive_message_chain(:esql, :query).and_return(response)
+        allow(response).to receive(:headers).and_return({})
+      end
+
+      it "includes 1st depth elements into event" do
+        esql_executor.do_run(output_queue, plugin_config["query"])
+
+        expect(output_queue.size).to eq(3)
+        3.times do
+          event = output_queue.pop
+          expect(event.get('time')).to eq(50)
+          expect(event.get('[time][min]')).to eq(nil)
+          expect(event.get('[time][max]')).to eq(nil)
+        end
+      end
+    end
+  end
+
+  describe "#column spec" do
+    let(:valid_spec) { { 'name' => 'field.name', 'type' => 'keyword' } }
+    let(:column_spec) { LogStash::Inputs::Elasticsearch::ColumnSpec.new(valid_spec) }
+
+    context "when initializes" do
+      it "sets the name and type attributes" do
+        expect(column_spec.name).to eq("field.name")
+        expect(column_spec.type).to eq("keyword")
+      end
+
+      it "freezes the name and type attributes" do
+        expect(column_spec.name).to be_frozen
+        expect(column_spec.type).to be_frozen
+      end
+    end
+
+    context "when calls the field reference" do
+      it "returns the correct field reference format" do
+        expect(column_spec.field_reference).to eq("[field][name]")
+      end
+    end
+  end
+end if LOGSTASH_VERSION >= LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION