logstash-output-elasticsearch 11.0.2-java → 11.0.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4d12d1aba765125efba841af202081a30171914e9b5dc6aca66772c546c3bd6
-  data.tar.gz: bbfe6f58cd3824fc67fd0c80a564f759a32863e0590a2749e772e1c05a9144aa
+  metadata.gz: 143b78ff484990a35c4d9a1b54ccb040bdba7e6aa2fd69928b0553544883fabd
+  data.tar.gz: a14e65b2b499d65d9a8f3389d0c0168f0447e1358338d867dd5b05a78ca28c13
 SHA512:
-  metadata.gz: 9ce4b4ddb86e640a3fa56325557db23b0fa19a4ae3692997c4c45ac97562507fd86c3b3369df761339ee22ff62390f70a83d9caeb4b31310fef8b79a42cd0f49
-  data.tar.gz: 9886cf6b18e0ed17aaa848376d979a31b4a3f90a55da17a2ffad997569d7847a08a137d3d62b62151a0012943715ded7572d718e41ce7d5227dbd025d117d44a
+  metadata.gz: fd6495287f9d5dff77d6006c5495013335eb869b7a9678973567358b9ce6ed772822cda5b38bfbeff1831e72406a4e550125cfafd9cbaeab20b588ccf584a7b6
+  data.tar.gz: d6eda43b3338ad401b889ff20f7bd246b3a44f921b0f006989572cb95145a518cb7103a1c23fd930018c2552316af33d356d4fecc8210ca95f0a6259a2b1037f

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.0.3
+ - Fixed SSL handshake hang indefinitely with proxy setup [#1032](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1032)
+
 ## 11.0.2
  - Validate that required functionality in Elasticsearch is available upon initial connection [#1015](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1015)
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -443,7 +443,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   def routing_field_name
-    maximum_seen_major_version >= 6 ? :routing : :_routing
+    :routing
   end
 
   # Determine the correct value for the 'type' field for the given event
@@ -456,9 +456,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
              event.sprintf(@document_type)
            else
              major_version = maximum_seen_major_version
-             if major_version < 6
-               es5_event_type(event)
-             elsif major_version == 6
+             if major_version == 6
                DEFAULT_EVENT_TYPE_ES6
              elsif major_version == 7
                DEFAULT_EVENT_TYPE_ES7
@@ -470,15 +468,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     type.to_s
   end
 
-  def es5_event_type(event)
-    type = event.get('type')
-    return DEFAULT_EVENT_TYPE_ES6 unless type
-    if !type.is_a?(String) && !type.is_a?(Numeric)
-      @logger.warn("Bad event type (non-string/integer type value set)", :type_class => type.class, :type_value => type, :event => event.to_hash)
-    end
-    type
-  end
-
   ##
   # WARNING: This method is overridden in a subclass in Logstash Core 7.7-7.8's monitoring,
   #          where a `client` argument is both required and ignored. In later versions of
@@ -487,7 +476,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @param noop_required_client [nil]: required `nil` for legacy reasons.
   # @return [Boolean]
   def use_event_type?(noop_required_client)
-    # always set type for ES <= 6
+    # always set type for ES 6
     # for ES 7 only set it if the user defined it
     (maximum_seen_major_version < 7) || (maximum_seen_major_version == 7 && @document_type)
   end

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -176,15 +176,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         @logger.warn("Sniff returned no nodes! Will not update hosts.")
         return nil
       else
-        case major_version(url_meta[:version])
-        when 5, 6, 7, 8
-          sniff_5x_and_above(nodes)
-        when 2, 1
-          sniff_2x_1x(nodes)
-        else
-          @logger.warn("Could not determine version for nodes in ES cluster!")
-          return nil
-        end
+        sniff(nodes)
       end
     end
     
@@ -192,7 +184,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       version_string.split('.').first.to_i
     end
     
-    def sniff_5x_and_above(nodes)
+    def sniff(nodes)
       nodes.map do |id,info|
         # Skip master-only nodes
         next if info["roles"] && info["roles"] == ["master"]
@@ -208,24 +200,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
-    def sniff_2x_1x(nodes)
-      nodes.map do |id,info|
-        # TODO Make sure this works with shield. Does that listed
-        # stuff as 'https_address?'
-        
-        addr_str = info['http_address'].to_s
-        next unless addr_str # Skip hosts with HTTP disabled
-
-        # Only connect to nodes that serve data
-        # this will skip connecting to client, tribe, and master only nodes
-        # Note that if 'attributes' is NOT set, then that's just a regular node
-        # with master + data + client enabled, so we allow that
-        attributes = info['attributes']
-        next if attributes && attributes['data'] == 'false'
-        address_str_to_uri(addr_str)
-      end.compact
-    end
-
     def stop_sniffer
       @sniffer.join if @sniffer
     end

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -16,20 +16,12 @@ module LogStash; module Outputs; class ElasticSearch
         begin
           if @ilm_enabled == 'auto'
             if ilm_on_by_default?
-              ilm_ready, error = ilm_ready?
-              if !ilm_ready
-                @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - #{error}")
-                false
-              else
-                ilm_alias_set?
-              end
+              ilm_alias_set?
             else
               @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - Your Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
               false
             end
           elsif @ilm_enabled.to_s == 'true'
-            ilm_ready, error = ilm_ready?
-            raise LogStash::ConfigurationError,"Index Lifecycle Management is set to enabled in Logstash, but cannot be used - #{error}"  unless ilm_ready
             ilm_alias_set?
           else
             false
@@ -47,29 +39,6 @@ module LogStash; module Outputs; class ElasticSearch
       maximum_seen_major_version >= 7
     end
 
-    def ilm_ready?
-      # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature
-      # available and enabled.
-      begin
-        xpack = client.get_xpack_info
-        features = xpack.nil? || xpack.empty? ? nil : xpack["features"]
-        ilm = features.nil? ? nil : features["ilm"]
-        return false, "Index Lifecycle management is not installed on your Elasticsearch cluster" if features.nil? || ilm.nil?
-        return false, "Index Lifecycle management is not available in your Elasticsearch cluster" unless ilm['available']
-        return false, "Index Lifecycle management is not enabled in your Elasticsearch cluster" unless ilm['enabled']
-        return true, nil
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible
-        if e.response_code == 404
-          return false, "Index Lifecycle management is not installed on your Elasticsearch cluster"
-        elsif e.response_code == 400
-          return false, "Index Lifecycle management is not installed on your Elasticsearch cluster"
-        else
-          raise e
-        end
-      end
-    end
-
     def default_index?(index)
       index == @default_index
     end
@@ -113,4 +82,4 @@ module LogStash; module Outputs; class ElasticSearch
       LogStash::Json.load(::IO.read(policy_path))
     end
   end
-end; end; end
+end; end; end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -53,7 +53,7 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.default_template_path(es_major_version, ecs_compatibility=:disabled)
-      template_version = es_major_version == 1 ? 2 : es_major_version
+      template_version = es_major_version
       default_template_name = "templates/ecs-#{ecs_compatibility}/elasticsearch-#{template_version}x.json"
       ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
     end

data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json

@@ -0,0 +1 @@
+lib/logstash/outputs/elasticsearch/templates/ecs-v1/Users/yaauie/src/elastic/logstash-plugins/logsta

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.0.2'
+  s.version         = '11.0.3'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency "manticore", '>= 0.7.1', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'

data/spec/es_spec_helper.rb

@@ -20,7 +20,9 @@ module ESHelper
   end
 
   def get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+    Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
+      allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
+    end
   end
 
   def doc_type
@@ -53,11 +55,7 @@ module ESHelper
   end
 
   def routing_field_name
-    if ESHelper.es_version_satisfies?(">=6")
-      :routing
-    else
-      :_routing
-    end
+    :routing
   end
 
   def self.es_version

data/spec/fixtures/_nodes/{5x_6x.json → 6x.json}

@@ -11,7 +11,7 @@
       "transport_address" : "http://localhost:9200",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "master"
@@ -29,7 +29,7 @@
       "transport_address" : "http://localhost:9201",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "data"
@@ -47,7 +47,7 @@
       "transport_address" : "http://localhost:9202",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "data",
@@ -66,7 +66,7 @@
       "transport_address" : "http://localhost:9203",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [ ],
       "http" : {
@@ -78,4 +78,4 @@
       }
     }
   }
-}
+}

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -8,62 +8,63 @@ RSpec::Matchers.define :a_valid_gzip_encoded_string do
   }
 end
 
-if ESHelper.es_version_satisfies?(">= 5")
-  describe "indexing with http_compression turned on", :integration => true do
-    let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
-    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
-    let(:event_count) { 10000 + rand(500) }
-    let(:events) { event_count.times.map { event }.to_a }
-    let(:config) {
-      {
-        "hosts" => get_host_port,
-        "index" => index,
-        "http_compression" => true
-      }
+describe "indexing with http_compression turned on", :integration => true do
+  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+  let(:event_count) { 10000 + rand(500) }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+    {
+      "hosts" => get_host_port,
+      "index" => index,
+      "http_compression" => true
     }
-    subject { LogStash::Outputs::ElasticSearch.new(config) }
+  }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-    let(:es_url) { "http://#{get_host_port}" }
-    let(:index_url) {"#{es_url}/#{index}"}
-    let(:http_client_options) { {} }
-    let(:http_client) do
-      Manticore::Client.new(http_client_options)
-    end
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{es_url}/#{index}"}
+  let(:http_client_options) { {} }
+  let(:http_client) do
+    Manticore::Client.new(http_client_options)
+  end
 
-    before do
-      subject.register
-      subject.multi_receive([])
-    end
+  before do
+    subject.register
+    subject.multi_receive([])
+  end
 
-    shared_examples "an indexer" do
-      it "ships events" do
-        subject.multi_receive(events)
+  shared_examples "an indexer" do
+    it "ships events" do
+      subject.multi_receive(events)
 
-        http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{es_url}/_refresh").call
 
-        response = http_client.get("#{index_url}/_count?q=*")
-        result = LogStash::Json.load(response.body)
-        cur_count = result["count"]
-        expect(cur_count).to eq(event_count)
+      response = http_client.get("#{index_url}/_count?q=*")
+      result = LogStash::Json.load(response.body)
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
 
-        response = http_client.get("#{index_url}/_search?q=*&size=1000")
-        result = LogStash::Json.load(response.body)
-        result["hits"]["hits"].each do |doc|
-          expect(doc["_type"]).to eq(type) if ESHelper.es_version_satisfies?(">= 6", "< 8")
-          expect(doc).not_to include("_type") if ESHelper.es_version_satisfies?(">= 8")
-          expect(doc["_index"]).to eq(index)
+      response = http_client.get("#{index_url}/_search?q=*&size=1000")
+      result = LogStash::Json.load(response.body)
+      result["hits"]["hits"].each do |doc|
+        if ESHelper.es_version_satisfies?("< 8")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
         end
+        expect(doc["_index"]).to eq(index)
       end
     end
+  end
 
-    it "sets the correct content-encoding header and body is compressed" do
-      expect(subject.client.pool.adapter.client).to receive(:send).
-        with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
-        and_call_original
-      subject.multi_receive(events)
-    end
-
-    it_behaves_like("an indexer")
+  it "sets the correct content-encoding header and body is compressed" do
+    expect(subject.client.pool.adapter.client).to receive(:send).
+      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+      and_call_original
+    subject.multi_receive(events)
   end
+
+  it_behaves_like("an indexer")
 end

data/spec/integration/outputs/delete_spec.rb

@@ -2,64 +2,62 @@ require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
 
-if ESHelper.es_version_satisfies?(">= 2")
-  describe "Versioned delete", :integration => true do
-    require "logstash/outputs/elasticsearch"
+describe "Versioned delete", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  let(:es) { get_client }
+
+  before :each do
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    es.indices.delete(:index => "*") rescue nil
+    es.indices.refresh
+  end
 
-    let(:es) { get_client }
+  context "when delete only" do
+    subject { LogStash::Outputs::ElasticSearch.new(settings) }
 
-    before :each do
-      # Delete all templates first.
-      # Clean ES of data before we start.
-      es.indices.delete_template(:name => "*")
-      # This can fail if there are no indexes, ignore failure.
-      es.indices.delete(:index => "*") rescue nil
-      es.indices.refresh
+    before do
+      subject.register
     end
 
-    context "when delete only" do
-      subject { LogStash::Outputs::ElasticSearch.new(settings) }
-
-      before do
-        subject.register
-      end
-
-      let(:settings) do
-        {
-          "manage_template" => true,
-          "index" => "logstash-delete",
-          "template_overwrite" => true,
-          "hosts" => get_host_port(),
-          "document_id" => "%{my_id}",
-          "version" => "%{my_version}",
-          "version_type" => "external",
-          "action" => "%{my_action}"
-        }
-      end
-
-      it "should ignore non-monotonic external version updates" do
-        id = "ev2"
-        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-        r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
-        expect(r['_version']).to eq(99)
-        expect(r['_source']['message']).to eq('foo')
+    let(:settings) do
+      {
+        "manage_template" => true,
+        "index" => "logstash-delete",
+        "template_overwrite" => true,
+        "hosts" => get_host_port(),
+        "document_id" => "%{my_id}",
+        "version" => "%{my_version}",
+        "version_type" => "external",
+        "action" => "%{my_action}"
+      }
+    end
 
-        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-        r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
-        expect(r2['_version']).to eq(99)
-        expect(r2['_source']['message']).to eq('foo')
-      end
+    it "should ignore non-monotonic external version updates" do
+      id = "ev2"
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r['_version']).to eq(99)
+      expect(r['_source']['message']).to eq('foo')
+
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
+      r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r2['_version']).to eq(99)
+      expect(r2['_source']['message']).to eq('foo')
+    end
 
-      it "should commit monotonic external version updates" do
-        id = "ev3"
-        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-        r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
-        expect(r['_version']).to eq(99)
-        expect(r['_source']['message']).to eq('foo')
+    it "should commit monotonic external version updates" do
+      id = "ev3"
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r['_version']).to eq(99)
+      expect(r['_source']['message']).to eq('foo')
 
-        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-        expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-      end
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
+      expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
     end
   end
 end