logstash-output-elasticsearch 11.0.0-java → 11.0.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd0e368ed484aa214da94fcdc6978f919b3139cf90f8db462aba17f9c1e86670
-  data.tar.gz: 67b475fdd703d50d7bbb806adebd46c3b6657ead3019e76c7517e3c2428335be
+  metadata.gz: f054da12af7761adfdd86d912602ad164e407d89a3965637763f51f341389185
+  data.tar.gz: b82ed842fa924dff112cc3b755f4bf08645f6e2ee39f820f39d5ce5f4700f6ed
 SHA512:
-  metadata.gz: f1e85fea62c9173d0ffdbf739487d8bcbbcfb41f304893e37333d22a8f42b0f527506d926c9f983480ef1c76eee43869ded32b84df98e19fdd999b9d5a26baa9
-  data.tar.gz: 0d1df95d541fa6e1d1161763051a3b4dc8fed10f62d878e7b4aa556d00c17bf6daf44a2fdc0c356c14c9c5431457112da601942dcf34dbf5386b05235933146e
+  metadata.gz: ea4d348ab10fc9caf3df58a9b13adc8b234d4e13834137006845507d74050f22499f6d957d2559436718ce76ced97ae4e11637d5f2da988f48ae7b8218db1118
+  data.tar.gz: ba9983c8a214de9793d30b3845c0f9a536ea2e67b909788d821262f7217d5127343d6dd0fb22a675895cc04aad2df9c723874f40248e87734a10204c8aec81d6

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 11.0.4
+ - [DOC] Clarify that `http_compression` applies to _requests_, and remove noise about _response_ decompression [#1000](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1000)
+
+## 11.0.3
+ - Fixed SSL handshake hang indefinitely with proxy setup [#1032](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1032)
+
+## 11.0.2
+ - Validate that required functionality in Elasticsearch is available upon initial connection [#1015](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1015)
+
+## 11.0.1
+ - Fix: DLQ regression shipped in 11.0.0 [#1012](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1012)
+ - [DOC] Fixed broken link in list item [#1011](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1011)
+
 ## 11.0.0
  - Feat: Data stream support [#988](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/988)
  - Refactor: reviewed logging format + restored ES (initial) setup error logging

data/docs/index.asciidoc

@@ -71,7 +71,7 @@ as logs, events, and metrics) and non-time series data in Elasticsearch.
 The data stream options are recommended for indexing time series datasets (such
 as logs, metrics, and events) into {es}:
 
-* <<plugins-{type}s-{plugin}-data_stream>> |<<string,string>>
+* <<plugins-{type}s-{plugin}-data_stream>>
 * <<plugins-{type}s-{plugin}-data_stream_auto_routing>> 
 * <<plugins-{type}s-{plugin}-data_stream_dataset>> 
 * <<plugins-{type}s-{plugin}-data_stream_namespace>>
@@ -278,16 +278,11 @@ not reevaluate its DNS value while the keepalive is in effect.
 
 ==== HTTP Compression
 
-This plugin supports request and response compression. Response compression is
-enabled by default for HTTP and for Elasticsearch versions 5.0 and later.
+This plugin supports request compression, and handles compressed responses
+from Elasticsearch.
 
-You don't have to set any configs in Elasticsearch for it to send back a
-compressed response. For versions before 5.0, or if HTTPS is enabled,
-`http.compression` must be set to `true` {ref}/modules-http.html#modules-http[in
-Elasticsearch] to take advantage of response compression when using this plugin.
-
-For requests compression, regardless of the Elasticsearch version, enable the
-`http_compression` setting in the Logstash config file.
+To enable request compression, use the <<plugins-{type}s-{plugin}-http_compression>>
+setting on this plugin.
 
 ==== Authentication
 
@@ -514,7 +509,7 @@ overwritten with a warning.
 * Default value is `logs`.
 
 The data stream type used to construct the data stream at index time.
-Currently, only `logs` and `metrics`are supported.
+Currently, only `logs`, `metrics` and `synthetics` are supported.
 
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`
@@ -640,8 +635,13 @@ Any special characters present in the URLs here MUST be URL escaped! This means
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-Enable gzip compression on requests. Note that response compression is on by
-default for Elasticsearch v5.0 and beyond
+Enable gzip compression on requests.
+
+This setting allows you to reduce this plugin's outbound network traffic by
+compressing each bulk _request_ to {es}.
+
+NOTE: This output plugin reads compressed _responses_ from {es} regardless
+      of the value of this setting.
 
 [id="plugins-{type}s-{plugin}-ilm_enabled"]
 ===== `ilm_enabled`

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -146,7 +146,7 @@ module LogStash module Outputs class ElasticSearch
     def data_stream_event_action_tuple(event)
       event_data = event.to_hash
       data_stream_event_sync(event_data) if data_stream_sync_fields
-      ['create', common_event_params(event), event_data] # action always 'create'
+      EventActionTuple.new('create', common_event_params(event), event, event_data)
     end
 
     DATA_STREAM_SYNC_FIELDS = [ 'type', 'dataset', 'namespace' ].freeze

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -176,15 +176,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         @logger.warn("Sniff returned no nodes! Will not update hosts.")
         return nil
       else
-        case major_version(url_meta[:version])
-        when 5, 6, 7, 8
-          sniff_5x_and_above(nodes)
-        when 2, 1
-          sniff_2x_1x(nodes)
-        else
-          @logger.warn("Could not determine version for nodes in ES cluster!")
-          return nil
-        end
+        sniff(nodes)
       end
     end
     
@@ -192,7 +184,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       version_string.split('.').first.to_i
     end
     
-    def sniff_5x_and_above(nodes)
+    def sniff(nodes)
       nodes.map do |id,info|
         # Skip master-only nodes
         next if info["roles"] && info["roles"] == ["master"]
@@ -208,24 +200,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
-    def sniff_2x_1x(nodes)
-      nodes.map do |id,info|
-        # TODO Make sure this works with shield. Does that listed
-        # stuff as 'https_address?'
-        
-        addr_str = info['http_address'].to_s
-        next unless addr_str # Skip hosts with HTTP disabled
-
-        # Only connect to nodes that serve data
-        # this will skip connecting to client, tribe, and master only nodes
-        # Note that if 'attributes' is NOT set, then that's just a regular node
-        # with master + data + client enabled, so we allow that
-        attributes = info['attributes']
-        next if attributes && attributes['data'] == 'false'
-        address_str_to_uri(addr_str)
-      end.compact
-    end
-
     def stop_sniffer
       @sniffer.join if @sniffer
     end

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -16,20 +16,12 @@ module LogStash; module Outputs; class ElasticSearch
         begin
           if @ilm_enabled == 'auto'
             if ilm_on_by_default?
-              ilm_ready, error = ilm_ready?
-              if !ilm_ready
-                @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - #{error}")
-                false
-              else
-                ilm_alias_set?
-              end
+              ilm_alias_set?
             else
               @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - Your Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
               false
             end
           elsif @ilm_enabled.to_s == 'true'
-            ilm_ready, error = ilm_ready?
-            raise LogStash::ConfigurationError,"Index Lifecycle Management is set to enabled in Logstash, but cannot be used - #{error}"  unless ilm_ready
             ilm_alias_set?
           else
             false
@@ -47,29 +39,6 @@ module LogStash; module Outputs; class ElasticSearch
       maximum_seen_major_version >= 7
     end
 
-    def ilm_ready?
-      # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature
-      # available and enabled.
-      begin
-        xpack = client.get_xpack_info
-        features = xpack.nil? || xpack.empty? ? nil : xpack["features"]
-        ilm = features.nil? ? nil : features["ilm"]
-        return false, "Index Lifecycle management is not installed on your Elasticsearch cluster" if features.nil? || ilm.nil?
-        return false, "Index Lifecycle management is not available in your Elasticsearch cluster" unless ilm['available']
-        return false, "Index Lifecycle management is not enabled in your Elasticsearch cluster" unless ilm['enabled']
-        return true, nil
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible
-        if e.response_code == 404
-          return false, "Index Lifecycle management is not installed on your Elasticsearch cluster"
-        elsif e.response_code == 400
-          return false, "Index Lifecycle management is not installed on your Elasticsearch cluster"
-        else
-          raise e
-        end
-      end
-    end
-
     def default_index?(index)
       index == @default_index
     end
@@ -113,4 +82,4 @@ module LogStash; module Outputs; class ElasticSearch
       LogStash::Json.load(::IO.read(policy_path))
     end
   end
-end; end; end
+end; end; end

data/lib/logstash/outputs/elasticsearch/license_checker.rb

@@ -11,7 +11,7 @@ module LogStash; module Outputs; class ElasticSearch
     # @param url [LogStash::Util::SafeURI] ES node URL
     # @return [Boolean] true if provided license is deemed appropriate
     def appropriate_license?(pool, url)
-      license = pool.get_license(url)
+      license = extract_license(pool.get_license(url))
       case license_status(license)
       when 'active'
         true
@@ -24,20 +24,26 @@ module LogStash; module Outputs; class ElasticSearch
       end
     end
 
+    NO_LICENSE = {}.freeze
+    private_constant :NO_LICENSE
+
+    def extract_license(license)
+      license.fetch("license", NO_LICENSE)
+    end
+
     def license_status(license)
-      license.fetch("license", {}).fetch("status", nil)
+      license.fetch("status", nil)
     end
 
     private
 
     def warn_no_license(url)
-      @logger.error("Connecting to an OSS distribution of Elasticsearch is no longer supported, " +
-                    "please upgrade to the default distribution of Elasticsearch", url: url.sanitized.to_s)
+      @logger.error("Could not connect to a compatible version of Elasticsearch", url: url.sanitized.to_s)
     end
 
     def warn_invalid_license(url, license)
-      @logger.warn("WARNING: Current Elasticsearch license is not active, " +
-                   "please check Elasticsearch's licensing information", url: url.sanitized.to_s, license: license)
+      @logger.warn("Elasticsearch license is not active, please check Elasticsearch’s licensing information",
+                   url: url.sanitized.to_s, license: license)
     end
 
   end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -53,7 +53,7 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.default_template_path(es_major_version, ecs_compatibility=:disabled)
-      template_version = es_major_version == 1 ? 2 : es_major_version
+      template_version = es_major_version
       default_template_name = "templates/ecs-#{ecs_compatibility}/elasticsearch-#{template_version}x.json"
       ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
     end

data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json

@@ -0,0 +1 @@
+lib/logstash/outputs/elasticsearch/templates/ecs-v1/Users/yaauie/src/elastic/logstash-plugins/logsta

data/lib/logstash/outputs/elasticsearch.rb

@@ -392,7 +392,21 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     params[:version] = event.sprintf(@version) if @version
     params[:version_type] = event.sprintf(@version_type) if @version_type
 
-    [action, params, event.to_hash]
+    EventActionTuple.new(action, params, event)
+  end
+
+  class EventActionTuple < Array # TODO: acting as an array for compatibility
+
+    def initialize(action, params, event, event_data = nil)
+      super(3)
+      self[0] = action
+      self[1] = params
+      self[2] = event_data || event.to_hash
+      @event = event
+    end
+
+    attr_reader :event
+
   end
 
   # @return Hash (initial) parameters for given event
@@ -429,7 +443,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   def routing_field_name
-    maximum_seen_major_version >= 6 ? :routing : :_routing
+    :routing
   end
 
   # Determine the correct value for the 'type' field for the given event
@@ -442,9 +456,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
              event.sprintf(@document_type)
            else
              major_version = maximum_seen_major_version
-             if major_version < 6
-               es5_event_type(event)
-             elsif major_version == 6
+             if major_version == 6
                DEFAULT_EVENT_TYPE_ES6
              elsif major_version == 7
                DEFAULT_EVENT_TYPE_ES7
@@ -456,15 +468,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     type.to_s
   end
 
-  def es5_event_type(event)
-    type = event.get('type')
-    return DEFAULT_EVENT_TYPE_ES6 unless type
-    if !type.is_a?(String) && !type.is_a?(Numeric)
-      @logger.warn("Bad event type (non-string/integer type value set)", :type_class => type.class, :type_value => type, :event => event.to_hash)
-    end
-    type
-  end
-
   ##
   # WARNING: This method is overridden in a subclass in Logstash Core 7.7-7.8's monitoring,
   #          where a `client` argument is both required and ignored. In later versions of
@@ -473,7 +476,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @param noop_required_client [nil]: required `nil` for legacy reasons.
   # @return [Boolean]
   def use_event_type?(noop_required_client)
-    # always set type for ES <= 6
+    # always set type for ES 6
     # for ES 7 only set it if the user defined it
     (maximum_seen_major_version < 7) || (maximum_seen_major_version == 7 && @document_type)
   end

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -200,8 +200,9 @@ module LogStash; module PluginMixins; module ElasticSearch
     def handle_dlq_status(message, action, status, response)
       # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
       if @dlq_writer
+        event, action = action.event, [action[0], action[1], action[2]]
         # TODO: Change this to send a map with { :status => status, :action => action } in the future
-        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
+        @dlq_writer.write(event, "#{message} status: #{status}, action: #{action}, response: #{response}")
       else
         if dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception'
           level = :error

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.0.0'
+  s.version         = '11.0.4'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency "manticore", '>= 0.7.1', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'

data/spec/es_spec_helper.rb

@@ -20,7 +20,9 @@ module ESHelper
   end
 
   def get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+    Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
+      allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
+    end
   end
 
   def doc_type
@@ -53,11 +55,7 @@ module ESHelper
   end
 
   def routing_field_name
-    if ESHelper.es_version_satisfies?(">=6")
-      :routing
-    else
-      :_routing
-    end
+    :routing
   end
 
   def self.es_version

data/spec/fixtures/_nodes/{5x_6x.json → 6x.json}

@@ -11,7 +11,7 @@
       "transport_address" : "http://localhost:9200",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "master"
@@ -29,7 +29,7 @@
       "transport_address" : "http://localhost:9201",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "data"
@@ -47,7 +47,7 @@
       "transport_address" : "http://localhost:9202",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "data",
@@ -66,7 +66,7 @@
       "transport_address" : "http://localhost:9203",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [ ],
       "http" : {
@@ -78,4 +78,4 @@
       }
     }
   }
-}
+}

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -8,62 +8,63 @@ RSpec::Matchers.define :a_valid_gzip_encoded_string do
   }
 end
 
-if ESHelper.es_version_satisfies?(">= 5")
-  describe "indexing with http_compression turned on", :integration => true do
-    let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
-    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
-    let(:event_count) { 10000 + rand(500) }
-    let(:events) { event_count.times.map { event }.to_a }
-    let(:config) {
-      {
-        "hosts" => get_host_port,
-        "index" => index,
-        "http_compression" => true
-      }
+describe "indexing with http_compression turned on", :integration => true do
+  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+  let(:event_count) { 10000 + rand(500) }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+    {
+      "hosts" => get_host_port,
+      "index" => index,
+      "http_compression" => true
     }
-    subject { LogStash::Outputs::ElasticSearch.new(config) }
+  }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-    let(:es_url) { "http://#{get_host_port}" }
-    let(:index_url) {"#{es_url}/#{index}"}
-    let(:http_client_options) { {} }
-    let(:http_client) do
-      Manticore::Client.new(http_client_options)
-    end
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{es_url}/#{index}"}
+  let(:http_client_options) { {} }
+  let(:http_client) do
+    Manticore::Client.new(http_client_options)
+  end
 
-    before do
-      subject.register
-      subject.multi_receive([])
-    end
+  before do
+    subject.register
+    subject.multi_receive([])
+  end
 
-    shared_examples "an indexer" do
-      it "ships events" do
-        subject.multi_receive(events)
+  shared_examples "an indexer" do
+    it "ships events" do
+      subject.multi_receive(events)
 
-        http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{es_url}/_refresh").call
 
-        response = http_client.get("#{index_url}/_count?q=*")
-        result = LogStash::Json.load(response.body)
-        cur_count = result["count"]
-        expect(cur_count).to eq(event_count)
+      response = http_client.get("#{index_url}/_count?q=*")
+      result = LogStash::Json.load(response.body)
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
 
-        response = http_client.get("#{index_url}/_search?q=*&size=1000")
-        result = LogStash::Json.load(response.body)
-        result["hits"]["hits"].each do |doc|
-          expect(doc["_type"]).to eq(type) if ESHelper.es_version_satisfies?(">= 6", "< 8")
-          expect(doc).not_to include("_type") if ESHelper.es_version_satisfies?(">= 8")
-          expect(doc["_index"]).to eq(index)
+      response = http_client.get("#{index_url}/_search?q=*&size=1000")
+      result = LogStash::Json.load(response.body)
+      result["hits"]["hits"].each do |doc|
+        if ESHelper.es_version_satisfies?("< 8")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
         end
+        expect(doc["_index"]).to eq(index)
       end
     end
+  end
 
-    it "sets the correct content-encoding header and body is compressed" do
-      expect(subject.client.pool.adapter.client).to receive(:send).
-        with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
-        and_call_original
-      subject.multi_receive(events)
-    end
-
-    it_behaves_like("an indexer")
+  it "sets the correct content-encoding header and body is compressed" do
+    expect(subject.client.pool.adapter.client).to receive(:send).
+      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+      and_call_original
+    subject.multi_receive(events)
   end
+
+  it_behaves_like("an indexer")
 end