logstash-input-elasticsearch 5.2.2 → 5.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7492197c24e31559240f18287c9ca4d45c8b9cf86aba977ee100bee6eec8ab1b
-  data.tar.gz: 7bc9bf7a118dd6594d299536341239c1a87484fc7b0cec617e33879a5a77d524
+  metadata.gz: 90dc46d1bcadf70173792a9c2319925ea5e78bd3e6322e7c323de178afb72fa4
+  data.tar.gz: 21548a5935b0aa5b153b6e1bc00c7f9f3f6ade7f6546a0beb0ce7cc6cef5a984
 SHA512:
-  metadata.gz: 41faaed2909e00cac9b58fa9347a5d53a81eaee27c4f4e548cec637bc37a448d63664918c4535f280b9df2202a9b494f372f97e2aee970e496f14620f70b5282
-  data.tar.gz: 997380a42e67e2f66bd60883ee1362842edb405607710a1a58241114277c5f2c7c1c3316c162920cbce6346554bac9d2d111dd777c79451f4c6a0edce364ee48
+  metadata.gz: 59e8a03c25d8ee9a1a02014b75d1096b81250b85c7f6bd7e78114764534d9ad244d5d758e974c84bc20d9aa0c8fad669a3ef636b17037426c30f2705d74083d2
+  data.tar.gz: 8b986365b25b8884e3aae581b37ce35f9190e50bf3ce4d17263887c31721efe16f16b97dcf2571dcb446991f7ea4406c6011a1d1dbd3afd7f3e07ec0c4b9b743

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 5.3.0
+  - Drop a support for Logstash 7.x by requiring elasticsearch gem >= 8. Logstash 8+ continues to work as before [#252](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/252)
+
 ## 5.2.2
-  - [DOC] Note that `search_after` requires permissions on underlying indices/data streams, not aliases [#251](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/TBD)
+  - [DOC] Note that `search_after` requires permissions on underlying indices/data streams, not aliases [#251](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/251)
 
 ## 5.2.1
   - Added support for encoded and non encoded api-key formats on plugin configuration [#237](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/237)

data/lib/logstash/inputs/elasticsearch.rb

@@ -13,6 +13,7 @@ require "logstash/plugin_mixins/normalize_config_support"
 require "base64"
 
 require "elasticsearch"
+require "elastic/transport/transport/http/manticore"
 require "manticore"
 
 # .Compatibility Note
@@ -350,7 +351,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     @client_options = {
       :hosts => hosts,
       :transport_options => transport_options,
-      :transport_class => get_transport_client_class,
+      :transport_class => ::Elastic::Transport::Transport::HTTP::Manticore,
       :ssl => ssl_options
     }
 
@@ -728,20 +729,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     last_run_metadata_path
   end
 
-  def get_transport_client_class
-    # LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport`
-    # And now `elasticsearch-transport` is old, instead we have `elastic-transport`.
-    # LS-core updated `elasticsearch` > 8: https://github.com/elastic/logstash/pull/17161
-    # Following source bits are for the compatibility to support both `elasticsearch-transport` and `elastic-transport` gems
-    require "elasticsearch/transport/transport/http/manticore"
-    require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
-    require_relative "elasticsearch/patches/_elasticsearch_transport_connections_selector"
-    ::Elasticsearch::Transport::Transport::HTTP::Manticore
-  rescue ::LoadError
-    require "elastic/transport/transport/http/manticore"
-    ::Elastic::Transport::Transport::HTTP::Manticore
-  end
-
   def validate_ls_version_for_esql_support!
     if Gem::Version.create(LOGSTASH_VERSION) < Gem::Version.create(LS_ESQL_SUPPORT_VERSION)
       fail("Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{LS_ESQL_SUPPORT_VERSION}")

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '5.2.2'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
 
-  s.add_runtime_dependency 'elasticsearch', '>= 7.17.9', '< 9'
+  s.add_runtime_dependency 'elasticsearch', '>= 8', '< 10'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
 

data/spec/inputs/elasticsearch_spec.rb

@@ -18,16 +18,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   let(:plugin) { described_class.new(config) }
   let(:queue) { Queue.new }
   let(:build_flavor) { "default" }
-  let(:es_version) { "7.5.0" }
+  let(:es_version) { "8.19.10" }
   let(:cluster_info) { {"version" => {"number" => es_version, "build_flavor" => build_flavor}, "tagline" => "You Know, for Search"} }
 
-  def elastic_ruby_v8_client_available?
-    Elasticsearch::Transport
-    false
-  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
-    true
-  end
-
   before(:each) do
     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
     allow_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(cluster_info)
@@ -36,7 +29,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   let(:base_config) do
     {
         'hosts' => ["localhost"],
-        'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+        'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+        'search_api' => 'scroll'
     }
   end
 
@@ -55,13 +49,13 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       it "does not set header Elastic-Api-Version" do
         plugin.register
         client = plugin.send(:client)
-        expect( extract_transport(client).options[:transport_options][:headers] ).not_to match hash_including("Elastic-Api-Version" => "2023-10-31")
+        expect( client.transport.options[:transport_options][:headers] ).not_to match hash_including("Elastic-Api-Version" => "2023-10-31")
       end
 
       it "sets an x-elastic-product-origin header identifying this as an internal plugin request" do
         plugin.register
         client = plugin.send(:client)
-        expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("x-elastic-product-origin"=>"logstash-input-elasticsearch")
+        expect( client.transport.options[:transport_options][:headers] ).to match hash_including("x-elastic-product-origin"=>"logstash-input-elasticsearch")
       end
     end
 
@@ -99,11 +93,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
         before do
           allow(Elasticsearch::Client).to receive(:new).and_return(es_client)
-          if elastic_ruby_v8_client_available?
-            allow(es_client).to receive(:info).and_raise(Elastic::Transport::Transport::Errors::BadRequest.new)
-          else
-            allow(es_client).to receive(:info).and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest.new)
-          end
+          allow(es_client).to receive(:info).and_raise(Elastic::Transport::Transport::Errors::BadRequest.new)
         end
 
         it "raises an exception" do
@@ -116,13 +106,13 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           expect_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(true)
           plugin.register
           client = plugin.send(:client)
-          expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("Elastic-Api-Version" => "2023-10-31")
+          expect( client.transport.options[:transport_options][:headers] ).to match hash_including("Elastic-Api-Version" => "2023-10-31")
         end
 
         it "sets an x-elastic-product-origin header identifying this as an internal plugin request" do
           plugin.register
           client = plugin.send(:client)
-          expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("x-elastic-product-origin"=>"logstash-input-elasticsearch")
+          expect( client.transport.options[:transport_options][:headers] ).to match hash_including("x-elastic-product-origin"=>"logstash-input-elasticsearch")
         end
       end
 
@@ -138,7 +128,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         it "sets custom headers" do
           plugin.register
           client = plugin.send(:client)
-          expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including(config["custom_headers"])
+          expect( client.transport.options[:transport_options][:headers] ).to match hash_including(config["custom_headers"])
         end
       end
     end
@@ -190,11 +180,12 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         "_type" => "logs",
         "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
         "_score" => 1.0,
-        "_source" => { "message" => ["ohayo"] }
+        "_source" => { "message" => ["ohayo"] },
+        "sort" => [1]
       }
       allow(@esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
-      allow(@esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
-      allow(@esclient).to receive(:clear_scroll).and_return(nil)
+      allow(@esclient).to receive(:open_point_in_time).and_return({"id" => "test-pit-id"})
+      allow(@esclient).to receive(:close_point_in_time).and_return(nil)
       allow(@esclient).to receive(:ping)
       allow(@esclient).to receive(:info).and_return(cluster_info)
     end
@@ -210,7 +201,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     let(:config) do
       {
           'hosts' => ["localhost"],
-          'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+          'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+          'search_api' => 'scroll'
       }
     end
 
@@ -269,7 +261,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         {
             'hosts' => ["localhost"],
             'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-            'target' => "[@metadata][_source]"
+            'target' => "[@metadata][_source]",
+            'search_api' => 'scroll'
         }
       end
 
@@ -293,7 +286,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           'query' => "#{LogStash::Json.dump(query)}",
           'slices' => slices,
           'docinfo' => true, # include ids
-          'docinfo_target' => '[@metadata]'
+          'docinfo_target' => '[@metadata]',
+          'search_api' => 'scroll'
       }
     end
     let(:query) do
@@ -753,13 +747,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:client)
-        target_field = :@seeds
-        begin
-          Elasticsearch::Transport::Client
-        rescue
-          target_field = :@hosts
-        end
-        expect( client.transport.instance_variable_get(target_field) ).to eql [{
+        expect( client.transport.instance_variable_get(:@hosts) ).to eql [{
                                                                               :scheme => "https",
                                                                               :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
                                                                               :port => 9243,
@@ -791,7 +779,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       it "should set authorization" do
         plugin.register
         client = plugin.send(:client)
-        auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+        auth_header = client.transport.options[:transport_options][:headers]['Authorization']
 
         expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
       end
@@ -831,7 +819,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           it "correctly sets the Authorization header" do
             plugin.register
             client = plugin.send(:client)
-            auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+            auth_header = client.transport.options[:transport_options][:headers]['Authorization']
 
             expect(auth_header).to eql("ApiKey #{encoded_api_key}")
           end
@@ -871,7 +859,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       it "should set proxy" do
         plugin.register
         client = plugin.send(:client)
-        proxy = extract_transport(client).options[:transport_options][:proxy]
+        proxy = client.transport.options[:transport_options][:proxy]
 
         expect( proxy ).to eql "http://localhost:1234"
       end
@@ -883,7 +871,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           plugin.register
           client = plugin.send(:client)
 
-          expect( extract_transport(client).options[:transport_options] ).to_not include(:proxy)
+          expect( client.transport.options[:transport_options] ).to_not include(:proxy)
         end
       end
     end
@@ -911,15 +899,15 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
                     "cluster_name": "docker-cluster",
                     "cluster_uuid": "DyR1hN03QvuCWXRy3jtb0g",
                     "version": {
-                        "number": "7.13.1",
+                        "number": "8.13.1",
                         "build_flavor": "default",
                         "build_type": "docker",
                         "build_hash": "9a7758028e4ea59bcab41c12004603c5a7dd84a9",
-                        "build_date": "2021-05-28T17:40:59.346932922Z",
+                        "build_date": "2024-05-28T17:40:59.346932922Z",
                         "build_snapshot": false,
-                        "lucene_version": "8.8.2",
-                        "minimum_wire_compatibility_version": "6.8.0",
-                        "minimum_index_compatibility_version": "6.0.0-beta1"
+                        "lucene_version": "9.10.0",
+                        "minimum_wire_compatibility_version": "7.17.0",
+                        "minimum_index_compatibility_version": "7.0.0"
                     },
                     "tagline": "You Know, for Search"
                 }
@@ -1020,8 +1008,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         let(:plugin) { described_class.new(config) }
         let(:event)  { LogStash::Event.new({}) }
 
-        # elasticsearch-ruby 7.17.9 initialize two user agent headers, `user-agent` and `User-Agent`
-        # hence, fail this header size test case
         xit "client should sent the expect user-agent" do
           plugin.register
 
@@ -1220,6 +1206,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     describe "scroll" do
+      let(:config) { super().merge({ "search_api" => "scroll" }) }
       let(:search_response) do
         {
           "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
@@ -1379,11 +1366,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
   end
 
-  # @note can be removed once we depends on elasticsearch gem >= 6.x
-  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
-    client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
-  end
-
   describe "#ESQL" do
     let(:config) do
       {

data/spec/inputs/elasticsearch_ssl_spec.rb

@@ -14,7 +14,7 @@ describe "SSL options" do
   before do
     allow(es_client_double).to receive(:close)
     allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object)
-    allow(es_client_double).to receive(:info).and_return({"version" => {"number" => "7.5.0", "build_flavor" => "default"},
+    allow(es_client_double).to receive(:info).and_return({"version" => {"number" => "8.19.0", "build_flavor" => "default"},
                                                           "tagline" => "You Know, for Search"})
     allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double)
   end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -77,11 +77,7 @@ describe LogStash::Inputs::Elasticsearch do
       it_behaves_like 'an elasticsearch index plugin'
 
       let(:unauth_exception_class) do
-        begin
-          Elasticsearch::Transport::Transport::Errors::Unauthorized
-        rescue
-          Elastic::Transport::Transport::Errors::Unauthorized
-        end
+        Elastic::Transport::Transport::Errors::Unauthorized
       end
 
       context "incorrect auth credentials" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.2.2
+  version: 5.3.0
 platform: ruby
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2026-05-01 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -91,20 +91,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.17.9
+        version: '8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '9'
+        version: '10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.17.9
+        version: '8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '9'
+        version: '10'
 - !ruby/object:Gem::Dependency
   name: logstash-mixin-ca_trusted_fingerprint_support
   requirement: !ruby/object:Gem::Requirement
@@ -266,8 +266,6 @@ files:
 - lib/logstash/inputs/elasticsearch/cursor_tracker.rb
 - lib/logstash/inputs/elasticsearch/esql.rb
 - lib/logstash/inputs/elasticsearch/paginated_search.rb
-- lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
-- lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/fixtures/test_certs/GENERATED_AT
@@ -305,7 +303,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.3
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:

data/lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb

@@ -1,51 +0,0 @@
-require 'elasticsearch'
-require 'elasticsearch/transport/transport/connections/selector'
-
-if Gem.loaded_specs['elasticsearch-transport'].version < Gem::Version.new("7.2.0")
-  # elasticsearch-transport versions prior to 7.2.0 suffered of a race condition on accessing
-  # the connection pool. This issue was fixed (in 7.2.0) with
-  # https://github.com/elastic/elasticsearch-ruby/commit/15f9d78591a6e8823948494d94b15b0ca38819d1
-  #
-  # This plugin, at the moment, is using elasticsearch >= 5.0.5
-  # When this requirement ceases, this patch could be removed.
-  module Elasticsearch
-    module Transport
-      module Transport
-        module Connections
-          module Selector
-
-            # "Round-robin" selector strategy (default).
-            #
-            class RoundRobin
-              include Base
-
-              # @option arguments [Connections::Collection] :connections Collection with connections.
-              #
-              def initialize(arguments = {})
-                super
-                @mutex = Mutex.new
-                @current = nil
-              end
-
-              # Returns the next connection from the collection, rotating them in round-robin fashion.
-              #
-              # @return [Connections::Connection]
-              #
-              def select(options={})
-                @mutex.synchronize do
-                  conns = connections
-                  if @current && (@current < conns.size-1)
-                    @current += 1
-                  else
-                    @current = 0
-                  end
-                  conns[@current]
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb

@@ -1,43 +0,0 @@
-# encoding: utf-8
-require "elasticsearch"
-require "elasticsearch/transport/transport/http/manticore"
-
-es_client_version = Gem.loaded_specs['elasticsearch-transport'].version
-if es_client_version >= Gem::Version.new('7.2') && es_client_version < Gem::Version.new('7.16')
-  # elasticsearch-transport 7.2.0 - 7.14.0 had a bug where setting http headers
-  #   ES::Client.new ..., transport_options: { headers: { 'Authorization' => ... } }
-  # would be lost https://github.com/elastic/elasticsearch-ruby/issues/1428
-  #
-  # NOTE: needs to be idempotent as filter ES plugin might apply the same patch!
-  #
-  # @private
-  module Elasticsearch
-    module Transport
-      module Transport
-        module HTTP
-          class Manticore
-
-            def apply_headers(request_options, options)
-              headers = (options && options[:headers]) || {}
-              headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
-
-              # this code is necessary to grab the correct user-agent header
-              # when this method is invoked with apply_headers(@request_options, options)
-              # from https://github.com/elastic/elasticsearch-ruby/blob/v7.14.0/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb#L113-L114
-              transport_user_agent = nil
-              if (options && options[:transport_options] && options[:transport_options][:headers])
-                transport_headers = options[:transport_options][:headers]
-                transport_user_agent = find_value(transport_headers, USER_AGENT_REGEX)
-              end
-
-              headers[USER_AGENT_STR] = transport_user_agent || find_value(headers, USER_AGENT_REGEX) || user_agent_header
-              headers[ACCEPT_ENCODING] = GZIP if use_compression?
-              (request_options[:headers] ||= {}).merge!(headers) # this line was changed
-            end
-
-          end
-        end
-      end
-    end
-  end
-end