logstash-output-elasticsearch 8.1.1-java → 8.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fbfa89ded2964b66fde5e29cd3155d8f2da08e03
-  data.tar.gz: 3c603e14218bc92384c4112527d38c834ff7fb3e
+  metadata.gz: 71b2fb654d86950996c5fae7c523295a1bc9a9cd
+  data.tar.gz: 2dd7cc43347238f9bbfa19d27b49368f6c162dc4
 SHA512:
-  metadata.gz: aea0668c8ac704c1db41f01f62bdf84a173776b8be7a55e04a6de6898121a3f89a3b94be2c7a94b1d75c17407926bb8c033627dcb1507a1fcbae4d1b04aa16c8
-  data.tar.gz: 23cce8e779627097c33cf6ebb59106f6353cd40fa9d786f67b74fa42655738646104dfdd1a5ddfc27e4609b6187fec3018eb540e0848c5862129662d5686fd59
+  metadata.gz: a5d577e3f5bc1b3fb4be51eb896f7b5eaa4ecd66a0b4ebb0ff1f37e5d92b555becad7b1ffcf88356deb0a2ec9162ae45aa4526078925057afb22e26ac3760d5d
+  data.tar.gz: 055311a5f0f991216851381a8f0c0da0480467b84942b64420ecc58d146351a51ab39151dd28a7e89ab12348bdcfde0fa488dfea4769a6cc6ab7c7e015b47d06

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 8.2.0
+  - Improved Elasticsearch version handling
+  - Improved event error logging when DLQ is disabled in Logstash
+
 ## 8.1.1
   - Retry all non-200 responses of the bulk API indefinitely
   - Improve documentation on retry codes

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -18,7 +18,7 @@ module LogStash; module Outputs; class ElasticSearch;
     def register
       @stopping = Concurrent::AtomicBoolean.new(false)
       # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
-      @dlq_writer = supports_dlq? ? execution_context.dlq_writer : nil
+      @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
       setup_hosts # properly sets @hosts
       build_client
@@ -280,8 +280,11 @@ module LogStash; module Outputs; class ElasticSearch;
       end
     end
 
-    def supports_dlq?
-      respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer)
+    def dlq_enabled?
+      # TODO there should be a better way to query if DLQ is enabled
+      # See more in: https://github.com/elastic/logstash/issues/8064
+      respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
+        !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
     end
   end
 end; end; end

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -82,9 +82,8 @@ module LogStash; module Outputs; class ElasticSearch;
       template_put(name, template)
     end
 
-    def get_version
-      url, response = @pool.get("")
-      LogStash::Json.load(response.body)["version"]
+    def connected_es_versions
+      @pool.connected_es_versions
     end
 
     def bulk(actions)
@@ -137,7 +136,7 @@ module LogStash; module Outputs; class ElasticSearch;
     def bulk_send(body_stream)
       params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
       # Discard the URL 
-      _, response = @pool.post(@bulk_path, params, body_stream.string)
+      response = @pool.post(@bulk_path, params, body_stream.string)
       if !body_stream.closed?
         body_stream.truncate(0)
         body_stream.seek(0)
@@ -153,12 +152,12 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def get(path)
-      url, response = @pool.get(path, nil)
+      response = @pool.get(path, nil)
       LogStash::Json.load(response.body)
     end
 
     def post(path, params = {}, body_string)
-      url, response = @pool.post(path, params, body_string)
+      response = @pool.post(path, params, body_string)
       LogStash::Json.load(response.body)
     end
 
@@ -329,15 +328,14 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def template_exists?(name)
-      url, response = @pool.head("/_template/#{name}")
+      response = @pool.head("/_template/#{name}")
       response.code >= 200 && response.code <= 299
     end
 
     def template_put(name, template)
       path = "_template/#{name}"
       logger.info("Installing elasticsearch template to #{path}")
-      url, response = @pool.put(path, nil, LogStash::Json.dump(template))
-      response
+      @pool.put(path, nil, LogStash::Json.dump(template))
     end
 
     # Build a bulk item for an elasticsearch update action

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -107,6 +107,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @state_mutex.synchronize { @url_info }
     end
 
+    def connected_es_versions
+      @state_mutex.synchronize do
+        @url_info.values.select {|v| v[:state] == :alive }.map {|v| v[:version] }
+      end
+    end
+
     def urls
       url_info.keys
     end
@@ -154,7 +160,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     ES2_SNIFF_RE_URL  = /([^\/]*)?\/?([^:]*):([0-9]+)/
     # Sniffs and returns the results. Does not update internal URLs!
     def check_sniff
-      _, resp = perform_request(:get, @sniffing_path)
+      _, url_meta, resp = perform_request(:get, @sniffing_path)
       parsed = LogStash::Json.load(resp.body)
       
       nodes = parsed['nodes']
@@ -162,12 +168,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         @logger.warn("Sniff returned no nodes! Will not update hosts.")
         return nil
       else
-        case major_version(nodes)
+        case major_version(url_meta[:version])
         when 5, 6
           sniff_5x_and_above(nodes)
-        when 2
-          sniff_2x_1x(nodes)
-        when 1
+        when 2, 1
           sniff_2x_1x(nodes)
         else
           @logger.warn("Could not determine version for nodes in ES cluster!")
@@ -176,8 +180,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
     
-    def major_version(nodes)
-      k,v = nodes.first; v['version'].split('.').first.to_i
+    def major_version(version_string)
+      version_string.split('.').first.to_i
     end
     
     def sniff_5x_and_above(nodes)
@@ -237,7 +241,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
           response = perform_request_to_url(url, :head, @healthcheck_path)
           # If no exception was raised it must have succeeded!
           logger.warn("Restored connection to ES instance", :url => url.sanitized.to_s)
-          @state_mutex.synchronize { meta[:state] = :alive }
+          # We reconnected to this node, check its ES version
+          es_version = get_es_version(url)
+          @state_mutex.synchronize do
+            meta[:version] = es_version
+            meta[:state] = :alive
+          end
         rescue HostUnreachableError, BadResponseCodeError => e
           logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: url.sanitized.to_s, error_type: e.class, error: e.message)
         end
@@ -253,15 +262,16 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def perform_request(method, path, params={}, body=nil)
-      with_connection do |url|
+      with_connection do |url, url_meta|
         resp = perform_request_to_url(url, method, path, params, body)
-        [url, resp]
+        [url, url_meta, resp]
       end
     end
 
     [:get, :put, :post, :delete, :patch, :head].each do |method|
       define_method(method) do |path, params={}, body=nil|
-        perform_request(method, path, params, body)
+        _, _, response = perform_request(method, path, params, body)
+        response
       end
     end
 
@@ -323,6 +333,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @state_mutex.synchronize { @url_info.size }
     end
 
+    def es_versions
+      @state_mutex.synchronize { @url_info.size }
+    end
+
     def add_url(url)
       @url_info[url] ||= empty_url_meta
     end
@@ -344,7 +358,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       # Custom error class used here so that users may retry attempts if they receive this error
       # should they choose to
       raise NoConnectionAvailableError, "No Available connections" unless url
-      yield url
+      yield url, url_meta
     rescue HostUnreachableError => e
       # Mark the connection as dead here since this is likely not transient
       mark_dead(url, e)
@@ -415,5 +429,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         end
       end
     end
+
+    def get_es_version(url)
+      request = perform_request_to_url(url, :get, ROOT_URI_PATH)
+      LogStash::Json.load(request.body)["version"]["number"]
+    end
   end
 end; end; end; end;

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -12,12 +12,10 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     private
-    def self.get_es_version(client)
-      client.get_version
-    end
-
     def self.get_es_major_version(client)
-      get_es_version(client)["number"][0]
+      # get the elasticsearch version of each node in the pool and
+      # pick the biggest major version
+      client.connected_es_versions.uniq.map {|version| version.split(".").first.to_i}.max
     end
 
     def self.get_template(path, es_major_version)
@@ -30,7 +28,7 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.default_template_path(es_major_version)
-      template_version = es_major_version == "1" ? "2" : es_major_version
+      template_version = es_major_version == 1 ? 2 : es_major_version
       default_template_name = "elasticsearch-template-es#{template_version}x.json"
       ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
     end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '8.1.1'
+  s.version         = '8.2.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -10,6 +10,12 @@ module ESHelper
   def get_client
     Elasticsearch::Client.new(:hosts => [get_host_port])
   end
+
+  def self.es_version_satisfies?(*requirement)
+    es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
+    es_release_version = Gem::Version.new(es_version).release
+    Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
+  end
 end
 
 RSpec.configure do |config|

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -9,58 +9,60 @@ RSpec::Matchers.define :a_valid_gzip_encoded_string do
   }
 end
 
-describe "indexing with http_compression turned on", :integration => true, :version_greater_than_equal_to_5x => true do
-  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
-  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-  let(:type) { 10.times.collect { rand(10).to_s }.join("") }
-  let(:event_count) { 10000 + rand(500) }
-  let(:events) { event_count.times.map { event }.to_a }
-  let(:config) {
-    {
-      "hosts" => get_host_port,
-      "index" => index,
-      "http_compression" => true
+if ESHelper.es_version_satisfies?(">= 5")
+  describe "indexing with http_compression turned on", :integration => true do
+    let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:events) { event_count.times.map { event }.to_a }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "http_compression" => true
+      }
     }
-  }
-  subject { LogStash::Outputs::ElasticSearch.new(config) }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-  let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
-  let(:http_client_options) { {} }
-  let(:http_client) do
-    Manticore::Client.new(http_client_options)
-  end
+    let(:es_url) { "http://#{get_host_port}" }
+    let(:index_url) {"#{es_url}/#{index}"}
+    let(:http_client_options) { {} }
+    let(:http_client) do
+      Manticore::Client.new(http_client_options)
+    end
 
-  before do
-    subject.register
-  end
-  
-  shared_examples "an indexer" do
-    it "ships events" do
-      subject.multi_receive(events)
+    before do
+      subject.register
+    end
 
-      http_client.post("#{es_url}/_refresh").call
+    shared_examples "an indexer" do
+      it "ships events" do
+        subject.multi_receive(events)
 
-      response = http_client.get("#{index_url}/_count?q=*")
-      result = LogStash::Json.load(response.body)
-      cur_count = result["count"]
-      expect(cur_count).to eq(event_count)
+        http_client.post("#{es_url}/_refresh").call
 
-      response = http_client.get("#{index_url}/_search?q=*&size=1000")
-      result = LogStash::Json.load(response.body)
-      result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
-        expect(doc["_index"]).to eq(index)
+        response = http_client.get("#{index_url}/_count?q=*")
+        result = LogStash::Json.load(response.body)
+        cur_count = result["count"]
+        expect(cur_count).to eq(event_count)
+
+        response = http_client.get("#{index_url}/_search?q=*&size=1000")
+        result = LogStash::Json.load(response.body)
+        result["hits"]["hits"].each do |doc|
+          expect(doc["_type"]).to eq(type)
+          expect(doc["_index"]).to eq(index)
+        end
       end
     end
-  end
 
-  it "sets the correct content-encoding header and body is compressed" do
-    expect(subject.client.pool.adapter.client).to receive(:send).
-      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
-      and_call_original
-    subject.multi_receive(events)
-  end
+    it "sets the correct content-encoding header and body is compressed" do
+      expect(subject.client.pool.adapter.client).to receive(:send).
+        with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+        and_call_original
+      subject.multi_receive(events)
+    end
 
-  it_behaves_like("an indexer")
+    it_behaves_like("an indexer")
+  end
 end

data/spec/integration/outputs/delete_spec.rb

@@ -2,62 +2,64 @@ require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
 
-describe "Versioned delete", :integration => true, :version_greater_than_equal_to_2x => true do
-  require "logstash/outputs/elasticsearch"
-
-  let(:es) { get_client }
-
-  before :each do
-    # Delete all templates first.
-    # Clean ES of data before we start.
-    es.indices.delete_template(:name => "*")
-    # This can fail if there are no indexes, ignore failure.
-    es.indices.delete(:index => "*") rescue nil
-    es.indices.refresh
-  end
+if ESHelper.es_version_satisfies?(">= 2")
+  describe "Versioned delete", :integration => true do
+    require "logstash/outputs/elasticsearch"
 
-  context "when delete only" do
-    subject { LogStash::Outputs::ElasticSearch.new(settings) }
+    let(:es) { get_client }
 
-    before do
-      subject.register
+    before :each do
+      # Delete all templates first.
+      # Clean ES of data before we start.
+      es.indices.delete_template(:name => "*")
+      # This can fail if there are no indexes, ignore failure.
+      es.indices.delete(:index => "*") rescue nil
+      es.indices.refresh
     end
 
-    let(:settings) do
-      {
-        "manage_template" => true,
-        "index" => "logstash-delete",
-        "template_overwrite" => true,
-        "hosts" => get_host_port(),
-        "document_id" => "%{my_id}",
-        "version" => "%{my_version}",
-        "version_type" => "external",
-        "action" => "%{my_action}"
-      }
-    end
+    context "when delete only" do
+      subject { LogStash::Outputs::ElasticSearch.new(settings) }
 
-    it "should ignore non-monotonic external version updates" do
-      id = "ev2"
-      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
-      expect(r['_version']).to eq(99)
-      expect(r['_source']['message']).to eq('foo')
-
-      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-      r2 = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
-      expect(r2['_version']).to eq(99)
-      expect(r2['_source']['message']).to eq('foo')
-    end
+      before do
+        subject.register
+      end
+
+      let(:settings) do
+        {
+          "manage_template" => true,
+          "index" => "logstash-delete",
+          "template_overwrite" => true,
+          "hosts" => get_host_port(),
+          "document_id" => "%{my_id}",
+          "version" => "%{my_version}",
+          "version_type" => "external",
+          "action" => "%{my_action}"
+        }
+      end
+
+      it "should ignore non-monotonic external version updates" do
+        id = "ev2"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+        r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+        expect(r['_version']).to eq(99)
+        expect(r['_source']['message']).to eq('foo')
+
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
+        r2 = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+        expect(r2['_version']).to eq(99)
+        expect(r2['_source']['message']).to eq('foo')
+      end
 
-    it "should commit monotonic external version updates" do
-      id = "ev3"
-      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
-      expect(r['_version']).to eq(99)
-      expect(r['_source']['message']).to eq('foo')
+      it "should commit monotonic external version updates" do
+        id = "ev3"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+        r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+        expect(r['_version']).to eq(99)
+        expect(r['_source']['message']).to eq('foo')
 
-      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-      expect { es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
+        expect { es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
     end
   end
 end