logstash-output-elasticsearch 10.8.6-java → 11.0.3-java

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -1,6 +1,5 @@
-require_relative "../../../spec/es_spec_helper"
+require_relative "../../../spec/spec_helper"
 require 'stud/temporary'
-require "logstash/outputs/elasticsearch"
 
 describe "SSL option" do
   let(:manticore_double) { double("manticoreSSL #{self.inspect}") }

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -12,6 +12,7 @@ describe "whitelisting error types in expected behavior" do
   before :each do
     allow(subject.logger).to receive(:warn)
     allow(subject).to receive(:maximum_seen_major_version).and_return(0)
+    allow(subject).to receive(:finish_register)
 
     subject.register
 
@@ -39,7 +40,7 @@ describe "whitelisting error types in expected behavior" do
 
   describe "when failure logging is enabled for everything" do
     it "should log a failure on the action" do
-      expect(subject.logger).to have_received(:warn).with("Failed action.", anything)
+      expect(subject.logger).to have_received(:warn).with("Failed action", anything)
     end
   end
 
@@ -47,7 +48,7 @@ describe "whitelisting error types in expected behavior" do
     let(:settings) { super().merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
 
     it "should log a failure on the action" do
-      expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything)
+      expect(subject.logger).not_to have_received(:warn).with("Failed action", anything)
     end
   end
 

data/spec/unit/outputs/license_check_spec.rb

@@ -14,22 +14,6 @@ describe LogStash::Outputs::ElasticSearch::LicenseChecker do
     end
   end
 
-  context "LicenseChecker API required by Pool specs" do
-    subject { described_class }
-
-    it "defines the oss? method" do
-      expect(subject.instance_methods).to include(:oss?)
-    end
-
-    it "defines the valid_es_license? method" do
-      expect(subject.instance_methods).to include(:valid_es_license?)
-    end
-
-    it "defines the log_license_deprecation_warn method" do
-      expect(subject.instance_methods).to include(:log_license_deprecation_warn)
-    end
-  end
-
   context "Pool class API required by the LicenseChecker" do
     subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool }
 

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 10.8.6
+  version: 11.0.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-06 00:00:00.000000000 Z
+date: 2021-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.7.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.7.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -50,20 +50,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  name: cabin
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -140,6 +126,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: cabin
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -170,6 +170,7 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/outputs/elasticsearch.rb
+- lib/logstash/outputs/elasticsearch/data_stream_support.rb
 - lib/logstash/outputs/elasticsearch/default-ilm-policy.json
 - lib/logstash/outputs/elasticsearch/http_client.rb
 - lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb
@@ -178,26 +179,21 @@ files:
 - lib/logstash/outputs/elasticsearch/ilm.rb
 - lib/logstash/outputs/elasticsearch/license_checker.rb
 - lib/logstash/outputs/elasticsearch/template_manager.rb
-- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json
-- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
 - lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
 - lib/logstash/plugin_mixins/elasticsearch/common.rb
 - lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
-- spec/fixtures/_nodes/2x_1x.json
-- spec/fixtures/_nodes/5x_6x.json
+- spec/fixtures/_nodes/6x.json
 - spec/fixtures/_nodes/7x.json
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
-- spec/fixtures/scripts/groovy/scripted_update.groovy
-- spec/fixtures/scripts/groovy/scripted_update_nested.groovy
-- spec/fixtures/scripts/groovy/scripted_upsert.groovy
 - spec/fixtures/scripts/painless/scripted_update.painless
 - spec/fixtures/scripts/painless/scripted_update_nested.painless
 - spec/fixtures/scripts/painless/scripted_upsert.painless
@@ -210,8 +206,8 @@ files:
 - spec/fixtures/test_certs/test.key
 - spec/integration/outputs/compressed_indexing_spec.rb
 - spec/integration/outputs/create_spec.rb
+- spec/integration/outputs/data_stream_spec.rb
 - spec/integration/outputs/delete_spec.rb
-- spec/integration/outputs/groovy_update_spec.rb
 - spec/integration/outputs/ilm_spec.rb
 - spec/integration/outputs/index_spec.rb
 - spec/integration/outputs/index_version_spec.rb
@@ -223,15 +219,16 @@ files:
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
-- spec/integration/outputs/templates_5x_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
+- spec/spec_helper.rb
 - spec/support/elasticsearch/api/actions/delete_ilm_policy.rb
 - spec/support/elasticsearch/api/actions/get_alias.rb
 - spec/support/elasticsearch/api/actions/get_ilm_policy.rb
 - spec/support/elasticsearch/api/actions/put_alias.rb
 - spec/support/elasticsearch/api/actions/put_ilm_policy.rb
 - spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/data_stream_support_spec.rb
 - spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb
 - spec/unit/outputs/elasticsearch/http_client/pool_spec.rb
 - spec/unit/outputs/elasticsearch/http_client_spec.rb
@@ -262,21 +259,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Stores logs in Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
-- spec/fixtures/_nodes/2x_1x.json
-- spec/fixtures/_nodes/5x_6x.json
+- spec/fixtures/_nodes/6x.json
 - spec/fixtures/_nodes/7x.json
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
-- spec/fixtures/scripts/groovy/scripted_update.groovy
-- spec/fixtures/scripts/groovy/scripted_update_nested.groovy
-- spec/fixtures/scripts/groovy/scripted_upsert.groovy
 - spec/fixtures/scripts/painless/scripted_update.painless
 - spec/fixtures/scripts/painless/scripted_update_nested.painless
 - spec/fixtures/scripts/painless/scripted_upsert.painless
@@ -289,8 +281,8 @@ test_files:
 - spec/fixtures/test_certs/test.key
 - spec/integration/outputs/compressed_indexing_spec.rb
 - spec/integration/outputs/create_spec.rb
+- spec/integration/outputs/data_stream_spec.rb
 - spec/integration/outputs/delete_spec.rb
-- spec/integration/outputs/groovy_update_spec.rb
 - spec/integration/outputs/ilm_spec.rb
 - spec/integration/outputs/index_spec.rb
 - spec/integration/outputs/index_version_spec.rb
@@ -302,15 +294,16 @@ test_files:
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
-- spec/integration/outputs/templates_5x_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
+- spec/spec_helper.rb
 - spec/support/elasticsearch/api/actions/delete_ilm_policy.rb
 - spec/support/elasticsearch/api/actions/get_alias.rb
 - spec/support/elasticsearch/api/actions/get_ilm_policy.rb
 - spec/support/elasticsearch/api/actions/put_alias.rb
 - spec/support/elasticsearch/api/actions/put_ilm_policy.rb
 - spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/data_stream_support_spec.rb
 - spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb
 - spec/unit/outputs/elasticsearch/http_client/pool_spec.rb
 - spec/unit/outputs/elasticsearch/http_client_spec.rb

data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json

@@ -1,95 +0,0 @@
-{
-  "template" : "logstash-*",
-  "settings" : {
-    "index.refresh_interval" : "5s"
-  },
-  "mappings" : {
-    "_default_" : {
-      "_all" : {"enabled" : true, "omit_norms" : true},
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "string", "index" : "analyzed", "omit_norms" : true,
-            "fielddata" : { "format" : "disabled" }
-          }
-        }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "string", "index" : "analyzed", "omit_norms" : true,
-            "fielddata" : { "format" : "disabled" },
-            "fields" : {
-              "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
-            }
-          }
-        }
-      }, {
-        "float_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "float",
-          "mapping" : { "type" : "float", "doc_values" : true }
-        }
-      }, {
-        "double_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "double",
-          "mapping" : { "type" : "double", "doc_values" : true }
-        }
-      }, {
-        "byte_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "byte",
-          "mapping" : { "type" : "byte", "doc_values" : true }
-        }
-      }, {
-        "short_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "short",
-          "mapping" : { "type" : "short", "doc_values" : true }
-        }
-      }, {
-        "integer_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "integer",
-          "mapping" : { "type" : "integer", "doc_values" : true }
-        }
-      }, {
-        "long_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "long",
-          "mapping" : { "type" : "long", "doc_values" : true }
-        }
-      }, {
-        "date_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "date",
-          "mapping" : { "type" : "date", "doc_values" : true }
-        }
-      }, {
-        "geo_point_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "geo_point",
-          "mapping" : { "type" : "geo_point", "doc_values" : true }
-        }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date", "doc_values" : true },
-        "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
-        "geoip"  : {
-          "type" : "object",
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip", "doc_values" : true },
-            "location" : { "type" : "geo_point", "doc_values" : true },
-            "latitude" : { "type" : "float", "doc_values" : true },
-            "longitude" : { "type" : "float", "doc_values" : true }
-          }
-        }
-      }
-    }
-  }
-}

data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json

@@ -1,46 +0,0 @@
-{
-  "template" : "logstash-*",
-  "version" : 50001,
-  "settings" : {
-    "index.refresh_interval" : "5s"
-  },
-  "mappings" : {
-    "_default_" : {
-      "_all" : {"enabled" : true, "norms" : false},
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text",
-            "norms" : false
-          }
-        }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text", "norms" : false,
-            "fields" : {
-              "keyword" : { "type": "keyword", "ignore_above": 256 }
-            }
-          }
-        }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date", "include_in_all": false },
-        "@version": { "type": "keyword", "include_in_all": false },
-        "geoip"  : {
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip" },
-            "location" : { "type" : "geo_point" },
-            "latitude" : { "type" : "half_float" },
-            "longitude" : { "type" : "half_float" }
-          }
-        }
-      }
-    }
-  }
-}

data/spec/fixtures/_nodes/2x_1x.json

@@ -1,27 +0,0 @@
-{
-  "cluster_name" : "dev",
-  "nodes" : {
-    "Ur_68iBvTlm7Xr1HgSsh6w" : {
-      "name" : "dev-es-master01",
-      "transport_address" : "http://localhost:9200",
-      "host" : "127.0.0.1",
-      "ip" : "127.0.0.1",
-      "version" : "2.4.6",
-      "build" : "5376dca"
-    },
-    "sari4do3RG-mgh2CIZeHwA" : {
-      "name" : "dev-es-data01",
-      "transport_address" : "http://localhost:9201",
-      "host" : "127.0.0.1",
-      "ip" : "127.0.0.1",
-      "version" : "2.4.6",
-      "build" : "5376dca",
-      "http_address" : "127.0.0.1:9201",
-      "http" : {
-        "bound_address" : [ "[::1]:9201", "127.0.0.1:9201" ],
-        "publish_address" : "127.0.0.1:9201",
-        "max_content_length_in_bytes" : 104857600
-      }
-    }
-  }
-}

data/spec/fixtures/scripts/groovy/scripted_update.groovy

@@ -1,2 +0,0 @@
-ctx._source.counter += event["count"]
-

data/spec/fixtures/scripts/groovy/scripted_update_nested.groovy

@@ -1,2 +0,0 @@
-ctx._source.counter += event["data"]["count"]
-

data/spec/fixtures/scripts/groovy/scripted_upsert.groovy

@@ -1,2 +0,0 @@
-ctx._source.counter = event["counter"]
-

data/spec/integration/outputs/groovy_update_spec.rb

@@ -1,150 +0,0 @@
-require_relative "../../../spec/es_spec_helper"
-
-if ESHelper.es_version_satisfies?('>= 2', '< 6')
-  describe "Update actions using groovy scripts", :integration => true, :update_tests => 'groovy' do
-    require "logstash/outputs/elasticsearch"
-
-    def get_es_output( options={} )
-      settings = {
-        "manage_template" => true,
-        "index" => "logstash-update",
-        "template_overwrite" => true,
-        "hosts" => get_host_port(),
-        "action" => "update",
-        "script_lang" => "groovy"
-      }
-      LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
-    end
-
-    before :each do
-      @es = get_client
-      # Delete all templates first.
-      # Clean ES of data before we start.
-      @es.indices.delete_template(:name => "*")
-      # This can fail if there are no indexes, ignore failure.
-      @es.indices.delete(:index => "*") rescue nil
-      @es.index(
-        :index => 'logstash-update',
-        :type => doc_type,
-        :id => "123",
-        :body => { :message => 'Test', :counter => 1 }
-      )
-      @es.indices.refresh
-    end
-
-    context "scripted updates" do
-      it "should increment a counter with event/doc 'count' variable" do
-        subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("count" => 2)])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(3)
-      end
-
-      it "should increment a counter with event/doc '[data][count]' nested variable" do
-        subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(4)
-      end
-
-      it "should increment a counter with event/doc 'count' variable with inline script" do
-        subject = get_es_output({
-          'document_id' => "123",
-          'script' => 'ctx._source.counter += event["counter"]',
-          'script_lang' => 'groovy',
-          'script_type' => 'inline'
-        })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(4)
-      end
-
-      it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-        subject = get_es_output({
-          'document_id' => "123",
-          'doc_as_upsert' => true,
-          'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
-          'script_lang' => 'groovy',
-          'script_type' => 'inline'
-        })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(4)
-      end
-
-      it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-        subject = get_es_output({
-          'document_id' => "456",
-          'doc_as_upsert' => true,
-          'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
-          'script_lang' => 'groovy',
-          'script_type' => 'inline'
-        })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(3)
-      end
-
-      it "should increment a counter with event/doc 'count' variable with indexed script" do
-        @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
-        subject = get_es_output({
-          'document_id' => "123",
-          'script' => 'indexed_update',
-          'script_lang' => 'groovy',
-          'script_type' => 'indexed'
-        })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("count" => 4 )])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(5)
-      end
-    end
-
-    context "when update with upsert" do
-      it "should create new documents with provided upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('upsert message')
-      end
-
-      it "should create new documents with event/doc as upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('sample message here')
-      end
-
-      it "should fail on documents with event/doc as upsert at external version" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
-    end
-
-    context "updates with scripted upsert" do
-      it "should create new documents with upsert content" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('upsert message')
-      end
-
-      it "should create new documents with event/doc as script params" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 1)])
-        @es.indices.refresh
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["counter"]).to eq(1)
-      end
-    end
-  end
-end