logstash-output-elasticsearch 10.2.1-java → 10.2.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4145b5bbd50a648c7557370139f600af5d5d83f193bcdf3956608b48b1d9d493
-  data.tar.gz: ac2d97688a709878bf8283905bee2b146694c73362739e6f47804c65ee183337
+  metadata.gz: cee8d8a64a98ebc0e90746c1e84323fbe842ae0afdbf072e530b50cbb4b27950
+  data.tar.gz: 2b24860b48ef18b5c28cfc9b62990ecc90bd202269b309b681181e0c18b9bae6
 SHA512:
-  metadata.gz: 6431afb36c0886342fc9930f931eaafca542d6c16c630082ad1acdb6603900cbc73652ef50248ec562b0192ca041604494167930254abf5f0d8412933132f7c8
-  data.tar.gz: f0b566a439e5e2a5ea9f6d0bf02a26853dcf8eb2879c00584ab0b58e20dc92139c7b479c4893c0a0ff608b484f6974046cc31e06187d056e3e7989dfe66a0ebd
+  metadata.gz: 07c5415be2991fdfed79ecbd27027272f9a59a72797b6a1efc0de021f9ae876e8949374a307da9663c3c36c6603ed8a8e4aedcf0009076354d0fe079572e2330
+  data.tar.gz: 47e5ca4b87727558c5c99a2e7134de3f7fec886b46f007f4af287e4ba91ee1e9fa71d578428da2149e42d2dce37ae0dd79f84684fb824f50004ed23b13345e1b

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 10.2.2
+  - Fixed 8.x type removal compatibility issue [#892](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/892) 
+
 ## 10.2.1
   - Fixed wording and corrected option in documentation [#881](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/881) [#883](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/883)
 

data/docs/index.asciidoc

@@ -298,13 +298,18 @@ Elasticsearch with the same ID.
   * There is no default value for this setting.
   * This option is deprecated
 
-Note: This option is deprecated due to the https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal of types in Elasticsearch 6.0].
+NOTE: This option is deprecated due to the https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal of types in Elasticsearch 6.0].
 It will be removed in the next major version of Logstash.
+
+NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`.
+
 This sets the document type to write events to. Generally you should try to write only
 similar events to the same 'type'. String expansion `%{foo}` works here.
 If you don't set a value for this option:
 
-- for elasticsearch clusters 6.x and above: the value of 'doc' will be used;
+- for elasticsearch clusters 8.x: no value will be used;
+- for elasticsearch clusters 7.x: the value of '_doc' will be used;
+- for elasticsearch clusters 6.x: the value of 'doc' will be used;
 - for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used.
 
 [id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"]

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -70,10 +70,11 @@ module LogStash; module Outputs; class ElasticSearch;
       params = {
         :_id => @document_id ? event.sprintf(@document_id) : nil,
         :_index => event.sprintf(@index),
-        :_type => get_event_type(event),
         routing_field_name => @routing ? event.sprintf(@routing) : nil
       }
 
+      params[:_type] = get_event_type(event) if client.maximum_seen_major_version < 8
+
       if @pipeline
         params[:pipeline] = event.sprintf(@pipeline)
       end
@@ -276,8 +277,10 @@ module LogStash; module Outputs; class ElasticSearch;
                  event.get("type") || DEFAULT_EVENT_TYPE_ES6
                elsif client.maximum_seen_major_version == 6
                  DEFAULT_EVENT_TYPE_ES6
-               else
+               elsif client.maximum_seen_major_version == 7
                  DEFAULT_EVENT_TYPE_ES7
+               else
+                 nil
                end
              end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.2.1'
+  s.version         = '10.2.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -27,13 +27,24 @@ module ESHelper
   end
 
   def doc_type
-    if ESHelper.es_version_satisfies?(">=7")
+    if ESHelper.es_version_satisfies?(">=8")
+      nil
+    elsif ESHelper.es_version_satisfies?(">=7")
       "_doc"
     else
       "doc"
     end
   end
 
+  def self.action_for_version(action)
+    action_params = action[1]
+    if ESHelper.es_version_satisfies?(">=8")
+      action_params.delete(:_type)
+    end
+    action[1] = action_params
+    action
+  end
+
   def todays_date
     Time.now.strftime("%Y.%m.%d")
   end
@@ -82,7 +93,6 @@ module ESHelper
     end
   end
 
-
   def self.es_version_satisfies?(*requirement)
     es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
     if es_version.nil?

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -50,9 +50,8 @@ if ESHelper.es_version_satisfies?(">= 5")
         response = http_client.get("#{index_url}/_search?q=*&size=1000")
         result = LogStash::Json.load(response.body)
         result["hits"]["hits"].each do |doc|
-          if ESHelper.es_version_satisfies?(">= 6")
-            expect(doc["_type"]).to eq(type)
-          end
+          expect(doc["_type"]).to eq(type) if ESHelper.es_version_satisfies?(">= 6", "< 8")
+          expect(doc).not_to include("_type") if ESHelper.es_version_satisfies?(">= 8")
           expect(doc["_index"]).to eq(index)
         end
       end

data/spec/integration/outputs/index_spec.rb

@@ -80,9 +80,8 @@ describe "indexing" do
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        if ESHelper.es_version_satisfies?(">= 6")
-          expect(doc["_type"]).to eq(type)
-        end
+        expect(doc["_type"]).to eq(type) if ESHelper.es_version_satisfies?(">= 6", "< 8")
+        expect(doc).not_to include("_type") if ESHelper.es_version_satisfies?(">= 8")
         expect(doc["_index"]).to eq(index)
       end
     end

data/spec/integration/outputs/retry_spec.rb

@@ -4,9 +4,9 @@ require_relative "../../../spec/es_spec_helper"
 describe "failures in bulk class expected behavior", :integration => true do
   let(:template) { '{"template" : "not important, will be updated by :index"}' }
   let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-  let(:action1) { ["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1] }
+  let(:action1) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1]) }
   let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-  let(:action2) { ["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2] }
+  let(:action2) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2]) }
   let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
 
   def mock_actions_with_response(*resp)

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -76,6 +76,44 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
+    describe "building an event action tuple" do
+      context "for 7.x elasticsearch clusters" do
+        let(:maximum_seen_major_version) { 7 }
+        it "should include '_type'" do
+          action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
+          action_params = action_tuple[1]
+          expect(action_params).to include(:_type => "_doc")
+        end
+
+        context "with 'document type set'" do
+          let(:options) { super.merge("document_type" => "bar")}
+          it "should get the event type from the 'document_type' setting" do
+            action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
+            action_params = action_tuple[1]
+            expect(action_params).to include(:_type => "bar")
+          end
+        end
+      end
+
+      context "for 8.x elasticsearch clusters" do
+        let(:maximum_seen_major_version) { 8 }
+        it "should not include '_type'" do
+          action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
+          action_params = action_tuple[1]
+          expect(action_params).not_to include(:_type)
+        end
+
+        context "with 'document type set'" do
+          let(:options) { super.merge("document_type" => "bar")}
+          it "should not include '_type'" do
+            action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
+            action_params = action_tuple[1]
+            expect(action_params).not_to include(:_type)
+          end
+        end
+      end
+    end
+
     describe "with auth" do
       let(:user) { "myuser" }
       let(:password) { ::LogStash::Util::Password.new("mypassword") }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 10.2.1
+  version: 10.2.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-09 00:00:00.000000000 Z
+date: 2019-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement