logstash-output-elasticsearch 11.0.1-java → 11.0.5-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: beee28e62f0872a041adc3c0f2c8546dbca09ced5c8a8e261cf1c9d07a0e64fe
-  data.tar.gz: f3f88dbbdf310557151a2f819ea566413d9f798be38d4717261c9dc544db97f1
+  metadata.gz: 2e275e2ca3a658814c730aaaa7464f81826dba01d8f2fb305859dc9aa33f0f34
+  data.tar.gz: f3dd23896923260bb9939e4ef90e56cf44c0daa0571b0987be3e39c45b9749c4
 SHA512:
-  metadata.gz: b43c800a66632c08a8c9c6d35367de3a127ebbf0abdfb0757cea7fb22501f4dc97e395e4b1ae2d57065c2c913a58e7f62dfa9a5cc4761c2f5c449d1f54448a4d
-  data.tar.gz: ff012fb8f3133de9f6002a35d733ea178828970f530c5c8f36239fdda605756f6d76b8a550690e2ac52aa64c551b73889453443ddc17f945a33b1f02be2abd0b
+  metadata.gz: 315cc2c6a4097f3923f76899c3bd5d58bb1ff779bc6aaaef4798cdf2db6a2d368ca0a8627e15e36e39b80484ad5bfea96ff7a9d77235c266627c9d4ef76f49c8
+  data.tar.gz: ae0591ff34ed9c88afd9b897c490e1787a28b68f44ff98816cd3ef5784cca71fd7d670c7aeee6b634ca01f837aaa2839198ec0fa8ed70454d2c0916e09e0293f

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 11.0.5
+ - Fixed running post-register action when Elasticsearch status change from unhealthy to healthy [#1035](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1035)
+
+## 11.0.4
+ - [DOC] Clarify that `http_compression` applies to _requests_, and remove noise about _response_ decompression [#1000](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1000)
+
+## 11.0.3
+ - Fixed SSL handshake hang indefinitely with proxy setup [#1032](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1032)
+
+## 11.0.2
+ - Validate that required functionality in Elasticsearch is available upon initial connection [#1015](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1015)
+
 ## 11.0.1
  - Fix: DLQ regression shipped in 11.0.0 [#1012](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1012)
  - [DOC] Fixed broken link in list item [#1011](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1011)

data/docs/index.asciidoc

@@ -278,16 +278,11 @@ not reevaluate its DNS value while the keepalive is in effect.
 
 ==== HTTP Compression
 
-This plugin supports request and response compression. Response compression is
-enabled by default for HTTP and for Elasticsearch versions 5.0 and later.
+This plugin supports request compression, and handles compressed responses
+from Elasticsearch.
 
-You don't have to set any configs in Elasticsearch for it to send back a
-compressed response. For versions before 5.0, or if HTTPS is enabled,
-`http.compression` must be set to `true` {ref}/modules-http.html#modules-http[in
-Elasticsearch] to take advantage of response compression when using this plugin.
-
-For requests compression, regardless of the Elasticsearch version, enable the
-`http_compression` setting in the Logstash config file.
+To enable request compression, use the <<plugins-{type}s-{plugin}-http_compression>>
+setting on this plugin.
 
 ==== Authentication
 
@@ -514,7 +509,7 @@ overwritten with a warning.
 * Default value is `logs`.
 
 The data stream type used to construct the data stream at index time.
-Currently, only `logs` and `metrics`are supported.
+Currently, only `logs`, `metrics` and `synthetics` are supported.
 
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`
@@ -640,8 +635,13 @@ Any special characters present in the URLs here MUST be URL escaped! This means
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-Enable gzip compression on requests. Note that response compression is on by
-default for Elasticsearch v5.0 and beyond
+Enable gzip compression on requests.
+
+This setting allows you to reduce this plugin's outbound network traffic by
+compressing each bulk _request_ to {es}.
+
+NOTE: This output plugin reads compressed _responses_ from {es} regardless
+      of the value of this setting.
 
 [id="plugins-{type}s-{plugin}-ilm_enabled"]
 ===== `ilm_enabled`

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -176,15 +176,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         @logger.warn("Sniff returned no nodes! Will not update hosts.")
         return nil
       else
-        case major_version(url_meta[:version])
-        when 5, 6, 7, 8
-          sniff_5x_and_above(nodes)
-        when 2, 1
-          sniff_2x_1x(nodes)
-        else
-          @logger.warn("Could not determine version for nodes in ES cluster!")
-          return nil
-        end
+        sniff(nodes)
       end
     end
     
@@ -192,7 +184,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       version_string.split('.').first.to_i
     end
     
-    def sniff_5x_and_above(nodes)
+    def sniff(nodes)
       nodes.map do |id,info|
         # Skip master-only nodes
         next if info["roles"] && info["roles"] == ["master"]
@@ -208,24 +200,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
-    def sniff_2x_1x(nodes)
-      nodes.map do |id,info|
-        # TODO Make sure this works with shield. Does that listed
-        # stuff as 'https_address?'
-        
-        addr_str = info['http_address'].to_s
-        next unless addr_str # Skip hosts with HTTP disabled
-
-        # Only connect to nodes that serve data
-        # this will skip connecting to client, tribe, and master only nodes
-        # Note that if 'attributes' is NOT set, then that's just a regular node
-        # with master + data + client enabled, so we allow that
-        attributes = info['attributes']
-        next if attributes && attributes['data'] == 'false'
-        address_str_to_uri(addr_str)
-      end.compact
-    end
-
     def stop_sniffer
       @sniffer.join if @sniffer
     end

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -92,6 +92,10 @@ module LogStash; module Outputs; class ElasticSearch;
       @pool.maximum_seen_major_version
     end
 
+    def alive_urls_count
+      @pool.alive_urls_count
+    end
+
     def bulk(actions)
       @action_count ||= 0
       @action_count += actions.size

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -16,20 +16,12 @@ module LogStash; module Outputs; class ElasticSearch
         begin
           if @ilm_enabled == 'auto'
             if ilm_on_by_default?
-              ilm_ready, error = ilm_ready?
-              if !ilm_ready
-                @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - #{error}")
-                false
-              else
-                ilm_alias_set?
-              end
+              ilm_alias_set?
             else
               @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - Your Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
               false
             end
           elsif @ilm_enabled.to_s == 'true'
-            ilm_ready, error = ilm_ready?
-            raise LogStash::ConfigurationError,"Index Lifecycle Management is set to enabled in Logstash, but cannot be used - #{error}"  unless ilm_ready
             ilm_alias_set?
           else
             false
@@ -47,29 +39,6 @@ module LogStash; module Outputs; class ElasticSearch
       maximum_seen_major_version >= 7
     end
 
-    def ilm_ready?
-      # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature
-      # available and enabled.
-      begin
-        xpack = client.get_xpack_info
-        features = xpack.nil? || xpack.empty? ? nil : xpack["features"]
-        ilm = features.nil? ? nil : features["ilm"]
-        return false, "Index Lifecycle management is not installed on your Elasticsearch cluster" if features.nil? || ilm.nil?
-        return false, "Index Lifecycle management is not available in your Elasticsearch cluster" unless ilm['available']
-        return false, "Index Lifecycle management is not enabled in your Elasticsearch cluster" unless ilm['enabled']
-        return true, nil
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible
-        if e.response_code == 404
-          return false, "Index Lifecycle management is not installed on your Elasticsearch cluster"
-        elsif e.response_code == 400
-          return false, "Index Lifecycle management is not installed on your Elasticsearch cluster"
-        else
-          raise e
-        end
-      end
-    end
-
     def default_index?(index)
       index == @default_index
     end
@@ -113,4 +82,4 @@ module LogStash; module Outputs; class ElasticSearch
       LogStash::Json.load(::IO.read(policy_path))
     end
   end
-end; end; end
+end; end; end

data/lib/logstash/outputs/elasticsearch/license_checker.rb

@@ -11,7 +11,7 @@ module LogStash; module Outputs; class ElasticSearch
     # @param url [LogStash::Util::SafeURI] ES node URL
     # @return [Boolean] true if provided license is deemed appropriate
     def appropriate_license?(pool, url)
-      license = pool.get_license(url)
+      license = extract_license(pool.get_license(url))
       case license_status(license)
       when 'active'
         true
@@ -24,20 +24,26 @@ module LogStash; module Outputs; class ElasticSearch
       end
     end
 
+    NO_LICENSE = {}.freeze
+    private_constant :NO_LICENSE
+
+    def extract_license(license)
+      license.fetch("license", NO_LICENSE)
+    end
+
     def license_status(license)
-      license.fetch("license", {}).fetch("status", nil)
+      license.fetch("status", nil)
     end
 
     private
 
     def warn_no_license(url)
-      @logger.error("Connecting to an OSS distribution of Elasticsearch is no longer supported, " +
-                    "please upgrade to the default distribution of Elasticsearch", url: url.sanitized.to_s)
+      @logger.error("Could not connect to a compatible version of Elasticsearch", url: url.sanitized.to_s)
     end
 
     def warn_invalid_license(url, license)
-      @logger.warn("WARNING: Current Elasticsearch license is not active, " +
-                   "please check Elasticsearch's licensing information", url: url.sanitized.to_s, license: license)
+      @logger.warn("Elasticsearch license is not active, please check Elasticsearch’s licensing information",
+                   url: url.sanitized.to_s, license: license)
     end
 
   end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -53,7 +53,7 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.default_template_path(es_major_version, ecs_compatibility=:disabled)
-      template_version = es_major_version == 1 ? 2 : es_major_version
+      template_version = es_major_version
       default_template_name = "templates/ecs-#{ecs_compatibility}/elasticsearch-#{template_version}x.json"
       ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
     end

data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json

@@ -0,0 +1 @@
+lib/logstash/outputs/elasticsearch/templates/ecs-v1/Users/yaauie/src/elastic/logstash-plugins/logsta

data/lib/logstash/outputs/elasticsearch.rb

@@ -443,7 +443,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   def routing_field_name
-    maximum_seen_major_version >= 6 ? :routing : :_routing
+    :routing
   end
 
   # Determine the correct value for the 'type' field for the given event
@@ -456,9 +456,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
              event.sprintf(@document_type)
            else
              major_version = maximum_seen_major_version
-             if major_version < 6
-               es5_event_type(event)
-             elsif major_version == 6
+             if major_version == 6
                DEFAULT_EVENT_TYPE_ES6
              elsif major_version == 7
                DEFAULT_EVENT_TYPE_ES7
@@ -470,15 +468,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     type.to_s
   end
 
-  def es5_event_type(event)
-    type = event.get('type')
-    return DEFAULT_EVENT_TYPE_ES6 unless type
-    if !type.is_a?(String) && !type.is_a?(Numeric)
-      @logger.warn("Bad event type (non-string/integer type value set)", :type_class => type.class, :type_value => type, :event => event.to_hash)
-    end
-    type
-  end
-
   ##
   # WARNING: This method is overridden in a subclass in Logstash Core 7.7-7.8's monitoring,
   #          where a `client` argument is both required and ignored. In later versions of
@@ -487,7 +476,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @param noop_required_client [nil]: required `nil` for legacy reasons.
   # @return [Boolean]
   def use_event_type?(noop_required_client)
-    # always set type for ES <= 6
+    # always set type for ES 6
     # for ES 7 only set it if the user defined it
     (maximum_seen_major_version < 7) || (maximum_seen_major_version == 7 && @document_type)
   end

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -128,8 +128,12 @@ module LogStash; module PluginMixins; module ElasticSearch
       client.maximum_seen_major_version
     end
 
+    def alive_urls_count
+      client.alive_urls_count
+    end
+
     def successful_connection?
-      !!maximum_seen_major_version
+      !!maximum_seen_major_version && alive_urls_count > 0
     end
 
     # launch a thread that waits for an initial successful connection to the ES cluster to call the given block

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.0.1'
+  s.version         = '11.0.5'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency "manticore", '>= 0.7.1', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'

data/spec/es_spec_helper.rb

@@ -20,7 +20,9 @@ module ESHelper
   end
 
   def get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+    Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
+      allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
+    end
   end
 
   def doc_type
@@ -53,11 +55,7 @@ module ESHelper
   end
 
   def routing_field_name
-    if ESHelper.es_version_satisfies?(">=6")
-      :routing
-    else
-      :_routing
-    end
+    :routing
   end
 
   def self.es_version

data/spec/fixtures/_nodes/{5x_6x.json → 6x.json}

@@ -11,7 +11,7 @@
       "transport_address" : "http://localhost:9200",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "master"
@@ -29,7 +29,7 @@
       "transport_address" : "http://localhost:9201",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "data"
@@ -47,7 +47,7 @@
       "transport_address" : "http://localhost:9202",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "data",
@@ -66,7 +66,7 @@
       "transport_address" : "http://localhost:9203",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [ ],
       "http" : {
@@ -78,4 +78,4 @@
       }
     }
   }
-}
+}

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -8,62 +8,63 @@ RSpec::Matchers.define :a_valid_gzip_encoded_string do
   }
 end
 
-if ESHelper.es_version_satisfies?(">= 5")
-  describe "indexing with http_compression turned on", :integration => true do
-    let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
-    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
-    let(:event_count) { 10000 + rand(500) }
-    let(:events) { event_count.times.map { event }.to_a }
-    let(:config) {
-      {
-        "hosts" => get_host_port,
-        "index" => index,
-        "http_compression" => true
-      }
+describe "indexing with http_compression turned on", :integration => true do
+  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+  let(:event_count) { 10000 + rand(500) }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+    {
+      "hosts" => get_host_port,
+      "index" => index,
+      "http_compression" => true
     }
-    subject { LogStash::Outputs::ElasticSearch.new(config) }
+  }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-    let(:es_url) { "http://#{get_host_port}" }
-    let(:index_url) {"#{es_url}/#{index}"}
-    let(:http_client_options) { {} }
-    let(:http_client) do
-      Manticore::Client.new(http_client_options)
-    end
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{es_url}/#{index}"}
+  let(:http_client_options) { {} }
+  let(:http_client) do
+    Manticore::Client.new(http_client_options)
+  end
 
-    before do
-      subject.register
-      subject.multi_receive([])
-    end
+  before do
+    subject.register
+    subject.multi_receive([])
+  end
 
-    shared_examples "an indexer" do
-      it "ships events" do
-        subject.multi_receive(events)
+  shared_examples "an indexer" do
+    it "ships events" do
+      subject.multi_receive(events)
 
-        http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{es_url}/_refresh").call
 
-        response = http_client.get("#{index_url}/_count?q=*")
-        result = LogStash::Json.load(response.body)
-        cur_count = result["count"]
-        expect(cur_count).to eq(event_count)
+      response = http_client.get("#{index_url}/_count?q=*")
+      result = LogStash::Json.load(response.body)
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
 
-        response = http_client.get("#{index_url}/_search?q=*&size=1000")
-        result = LogStash::Json.load(response.body)
-        result["hits"]["hits"].each do |doc|
-          expect(doc["_type"]).to eq(type) if ESHelper.es_version_satisfies?(">= 6", "< 8")
-          expect(doc).not_to include("_type") if ESHelper.es_version_satisfies?(">= 8")
-          expect(doc["_index"]).to eq(index)
+      response = http_client.get("#{index_url}/_search?q=*&size=1000")
+      result = LogStash::Json.load(response.body)
+      result["hits"]["hits"].each do |doc|
+        if ESHelper.es_version_satisfies?("< 8")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
         end
+        expect(doc["_index"]).to eq(index)
       end
     end
+  end
 
-    it "sets the correct content-encoding header and body is compressed" do
-      expect(subject.client.pool.adapter.client).to receive(:send).
-        with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
-        and_call_original
-      subject.multi_receive(events)
-    end
-
-    it_behaves_like("an indexer")
+  it "sets the correct content-encoding header and body is compressed" do
+    expect(subject.client.pool.adapter.client).to receive(:send).
+      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+      and_call_original
+    subject.multi_receive(events)
   end
+
+  it_behaves_like("an indexer")
 end