logstash-output-elasticsearch 9.4.0-java → 10.0.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a6c544ed5e43a7c722849336fedcd792b7f843e3ed12de12a2898d019a56084
-  data.tar.gz: 9db3b25912b6d104752a6eb73cc4d12fe900a41b7fad369720fbb56cf0d087cf
+  metadata.gz: 5437ab2d215d988d230d11363dddb6759fca10691dc89c4bb6fb64ee9963e125
+  data.tar.gz: 9815fd0ac531dd9d345045a4fbcd912c8a552f49abcccb01c5ec97e223ef907b
 SHA512:
-  metadata.gz: f785bc20eaad291c9a5eb30e4aee4737ce55dbcc879267b71fe0a1bc13e938410374080c9c28094ae28406894890a7710b04fa9eea5aab1ec854efd18db5aae2
-  data.tar.gz: 50aef5de97548888460009d7148530b65b9b5ea62ae49bc7ee4ec31935e51bb7d4a870690d23015738682072c9c5fadac2390af19a633a5702f6dd1aef60793c
+  metadata.gz: 5f9273bdbcc946ee9fa7b646aa8a924432e618f2b5cfa5def541ffed237e7b365fd68a36b486cc6d30f049e3d1f7a40d31322fbcfca03c60be6fe2c38ff7020d
+  data.tar.gz: d607546d21651f80d4f0207d2e967ef03520a7faa40be4cc66507e972a6bf5b7763e9a701d2b9d5e98f3587a9bc4a9bca509789f545908d6b2fa01be1be4dd72

data/CHANGELOG.md

@@ -1,5 +1,11 @@
-## 9.4.0
-  - Added 'auto' setting for ILM [#841](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/841)
+## 10.0.1
+  - Reverted `document_type` obsoletion [#844](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/844)
+
+## 10.0.0 (yanked due to issues with document_type obsoletion)
+  - Changed deprecated `document_type` option to obsolete [#824](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/824)
+  - Remove support for parent child (still support join data type) since we don't support multiple document types any more
+  - Removed obsolete `flush_size` and `idle_flush_time`
+  - Added 'auto' setting for ILM with default of 'auto' [#838](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/838)
 
 ## 9.3.2
   - Fixed sniffing support for 7.x [#827](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/827)

data/docs/index.asciidoc

@@ -105,20 +105,11 @@ This feature requires an Elasticsearch instance of 6.6.0 or higher with at least
 
 Logstash can use {ref}/index-lifecycle-management.html[Index Lifecycle Management] to automate the management of indices over time.
 
-The use of Index Lifecycle Management is controlled by the `ilm_enabled` setting.
-The `ilm_enabled` flag can be set to `true`, `false` (default) or `auto`.
+The use of Index Lifecycle Management is controlled by the `ilm_enabled` setting. By default, this is will
+automatically detect whether the Elasticsearch instance supports ILM, and will use it if it is available. `ilm_enabled`
+can also be set to `true` or `false` to override the automatic detection, or disable ILM.
 
-The use of Index Lifecycle Management is controlled by the `ilm_enabled` setting.
-The `ilm_enabled` flag can be set to `true`, `false` (default) or `auto`.
-
-Setting `ilm_enabled` to `auto` will automatically detect whether the
-Elasticsearch instance is version `7.0.0` and above and has Index Lifecycle Management enabled, and will enable its usage if so.
-
-Setting `ilm_enabled` to `true` will enable the Index Lifecycle Management feature if it is available on the
-Elasticsearch cluster - the plugin will not start if `ilm_enabled` is set to `true` and the Elasticsearch cluster does
-not support ILM.
-
-Enabling ILM support will overwrite the index settings and adjust the Logstash template to write the necessary settings for the template
+This will overwrite the index settings and adjust the Logstash template to write the necessary settings for the template
 to support index lifecycle management, including the index policy and rollover alias to be used.
 
 Logstash will create a rollover alias for the indices to be written to, including a pattern for how the actual indices will be named, and unless an ILM policy that already exists has been specified,
@@ -135,10 +126,9 @@ See config below for an example:
 [source,ruby]
     output {
       elasticsearch {
-        ilm_enabled => true
-        ilm_rollover_alias => "custom"
-        ilm_pattern => "000001"
-        ilm_policy => "custom_policy"
+        ilm_rollover_alias: "custom"
+        ilm_pattern: "000001"
+        ilm_policy: "custom_policy"
       }
     }
 
@@ -378,16 +368,15 @@ Enable gzip compression on requests. Note that response compression is on by def
 ===== `ilm_enabled`
 
   * Value can be any of: `true`, `false`, `auto`
-  * Default value is `false`
+  * Default value is `auto`
 
-The default setting of `false` will disable the Index Lifecycle Management feature.
+The default setting of `auto` will automatically enable the Index Lifecycle Management feature, if the Elasticsearch
+ cluster is running Elasticsearch version `7.0.0` or higher with the ILM feature enabled, and disable it otherwise.
 
+Setting this flag to `false` will disable the Index Lifecycle Management feature, even if the Elasticsearch cluster supports ILM.
 Setting this flag to `true` will enable Index Lifecycle Management feature, if the Elasticsearch cluster supports it. This is required
 to enable Index Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`.
 
-Setting this flag to `auto` will automatically enable the Index Lifecycle Management feature, if the Elasticsearch
- cluster is running Elasticsearch version `7.0.0` or higher with the ILM feature enabled, and disable it otherwise.
-
 NOTE: This feature requires a Basic License or above to be installed on an Elasticsearch cluster version 6.6.0 or later
 
 [id="plugins-{type}s-{plugin}-ilm_pattern"]

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -97,10 +97,6 @@ module LogStash; module Outputs; class ElasticSearch
       # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
       mod.config :hosts, :validate => :uri, :default => [::LogStash::Util::SafeURI.new("//127.0.0.1")], :list => true
 
-      mod.config :flush_size, :validate => :number, :obsolete => "This setting is no longer available as we now try to restrict bulk requests to sane sizes. See the 'Batch Sizes' section of the docs. If you think you still need to restrict payloads based on the number, not size, of events, please open a ticket."
-
-      mod.config :idle_flush_time, :validate => :number, :obsolete => "This settings is no longer valid. This was a no-op now as every pipeline batch is flushed synchronously obviating the need for this option."
-
       # Set upsert content for update mode.s
       # Create a new document with this parameter as json string if `document_id` doesn't exists
       mod.config :upsert, :validate => :string, :default => ""
@@ -147,7 +143,7 @@ module LogStash; module Outputs; class ElasticSearch
       # ILM configurations (beta)
       # -----
       # Flag for enabling Index Lifecycle Management integration.
-      mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => false
+      mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto'
 
       # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
       mod.config :ilm_rollover_alias, :validate => :string, :default => DEFAULT_ROLLOVER_ALIAS

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.4.0'
+  s.version         = '10.0.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -12,7 +12,7 @@ if ESHelper.es_version_satisfies?(">= 5")
   describe "indexing with http_compression turned on", :integration => true do
     let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
     let(:event_count) { 10000 + rand(500) }
     let(:events) { event_count.times.map { event }.to_a }
     let(:config) {
@@ -50,8 +50,6 @@ if ESHelper.es_version_satisfies?(">= 5")
         result = LogStash::Json.load(response.body)
         result["hits"]["hits"].each do |doc|
           if ESHelper.es_version_satisfies?(">= 6")
-            expect(doc["_type"]).to eq(doc_type)
-          else
             expect(doc["_type"]).to eq(type)
           end
           expect(doc["_index"]).to eq(index)

data/spec/integration/outputs/ilm_spec.rb

@@ -486,7 +486,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       if ESHelper.es_version_satisfies?(">=7.0")
         context 'when Elasticsearch is version 7 or above' do
-          it_behaves_like 'an ILM disabled Logstash'
+          it_behaves_like 'an ILM enabled Logstash'
         end
       end
 

data/spec/integration/outputs/index_spec.rb

@@ -12,7 +12,8 @@ describe "TARGET_BULK_BYTES", :integration => true do
       }
   }
   let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-  let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+
   subject { LogStash::Outputs::ElasticSearch.new(config) }
 
   before do
@@ -47,7 +48,7 @@ end
 describe "indexing" do
   let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
   let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-  let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
   let(:event_count) { 1 + rand(2) }
   let(:config) { "not implemented" }
   let(:events) { event_count.times.map { event }.to_a }
@@ -79,8 +80,6 @@ describe "indexing" do
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
         if ESHelper.es_version_satisfies?(">= 6")
-          expect(doc["_type"]).to eq(doc_type)
-        else
           expect(doc["_type"]).to eq(type)
         end
         expect(doc["_index"]).to eq(index)
@@ -125,7 +124,7 @@ describe "indexing" do
   end
 
   describe "an indexer with no type value set (default to doc)", :integration => true do
-    let(:type) { "doc" }
+    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
     let(:config) {
       {
         "hosts" => get_host_port,

data/spec/integration/outputs/parent_spec.rb

@@ -1,89 +1,13 @@
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
-if ESHelper.es_version_satisfies?("<= 5.x")
-  context "when using elasticsearch 5.x and before", :integration => true do
-    shared_examples "a type based parent indexer" do
-      let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-      let(:type) { 10.times.collect { rand(10).to_s }.join("") }
-      let(:event_count) { 10000 + rand(500) }
-      let(:parent) { "not_implemented" }
-      let(:config) { "not_implemented" }
-      let(:default_headers) {
-        {"Content-Type" => "application/json"}
-      }
-      subject { LogStash::Outputs::ElasticSearch.new(config) }
-
-      before do
-        # Add mapping and a parent document
-        index_url = "http://#{get_host_port()}/#{index}"
-        mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
-        Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
-        pdoc = { "foo" => "bar" }
-        Manticore.put("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers}).call
-
-        subject.register
-        subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
-      end
-
-
-      it "ships events" do
-        index_url = "http://#{get_host_port()}/#{index}"
-
-        Manticore.post("#{index_url}/_refresh").call
-
-        # Wait until all events are available.
-        Stud::try(10.times) do
-          query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
-          response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
-          data = response.body
-          result = LogStash::Json.load(data)
-          cur_count = result["count"]
-          expect(cur_count).to eq(event_count)
-        end
-      end
-    end
-
-    describe "(http protocol) index events with static parent" do
-      it_behaves_like 'a type based parent indexer' do
-        let(:parent) { "test" }
-        let(:config) {
-          {
-            "hosts" => get_host_port,
-            "index" => index,
-            "parent" => parent
-          }
-        }
-      end
-    end
-
-    describe "(http_protocol) index events with fieldref in parent value" do
-      it_behaves_like 'a type based parent indexer' do
-        let(:config) {
-          {
-            "hosts" => get_host_port,
-            "index" => index,
-            "parent" => "%{link_to}"
-          }
-        }
-      end
-    end
-  end
-end
-
 if ESHelper.es_version_satisfies?(">= 5.6")
   context "when using elasticsearch 5.6 and above", :integration => true do
 
     shared_examples "a join field based parent indexer" do
       let(:index) { 10.times.collect { rand(10).to_s }.join("") }
 
-      let(:type) do
-        if ESHelper.es_version_satisfies?('<7')
-          10.times.collect { rand(10).to_s }.join("")
-        else
-          "_doc"
-        end
-      end
+      let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
 
       let(:event_count) { 10000 + rand(500) }
       let(:parent) { "not_implemented" }

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -74,25 +74,6 @@ describe LogStash::Outputs::ElasticSearch do
           expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar")
         end
       end
-
-      context "with a bad type event field in a < 6.0 es cluster" do
-        let(:maximum_seen_major_version) { 5 }
-        let(:type_arg) { ["foo"] }
-        let(:result) { subject.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
-
-        before do
-          allow(subject.instance_variable_get(:@logger)).to receive(:warn)
-          result
-        end
-
-        it "should call @logger.warn and return nil" do
-          expect(subject.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
-        end
-
-        it "should set the type to the stringified value" do
-          expect(result).to eql(type_arg.to_s)
-        end
-      end
     end
 
     describe "with auth" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 9.4.0
+  version: 10.0.1
 platform: java
 authors:
 - Elastic