logstash-output-elasticsearch 10.6.1-java → 10.8.1-java

data/spec/integration/outputs/metrics_spec.rb

@@ -19,11 +19,7 @@ describe "metrics", :integration => true do
 
     # Clean ES of data before we start.
     @es = get_client
-    @es.indices.delete_template(:name => "*")
-
-    # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
-    #@es.indices.refresh
+    clean(@es)
     subject.register
   end
 

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -15,8 +15,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
   let(:manticore_double) { double("manticore a") }
   before(:each) do
+    stub_const('LogStash::OSS', oss)
 
-    allow(::LogStash::Outputs::ElasticSearch).to receive(:oss?).and_return(oss)
     response_double = double("manticore response").as_null_object
     # Allow healtchecks
     allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
@@ -26,8 +26,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
     allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
-    allow(subject).to receive(:oss?).and_return(oss)
-    allow(subject).to receive(:valid_es_license?).and_return(valid_license)
+    allow(subject.license_checker).to receive(:oss?).and_return(oss)
+    allow(subject.license_checker).to receive(:valid_es_license?).and_return(valid_license)
   end
 
   after do
@@ -245,27 +245,67 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     before(:each) do
       allow(subject).to receive(:health_check_request)
     end
+
+    let(:options) do
+      super.merge(:license_checker => license_checker)
+    end
+
+    context 'when LicenseChecker#acceptable_license? returns false' do
+      let(:license_checker) { double('LicenseChecker', :appropriate_license? => false) }
+
+      it 'does not mark the URL as active' do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(0)
+      end
+    end
+
+    context 'when LicenseChecker#acceptable_license? returns true' do
+      let(:license_checker) { double('LicenseChecker', :appropriate_license? => true) }
+
+      it 'marks the URL as active' do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(1)
+      end
+    end
+  end
+
+  # TODO: extract to ElasticSearchOutputLicenseChecker unit spec
+  describe "license checking with ElasticSearchOutputLicenseChecker" do
+    let(:options) do
+      super().merge(:license_checker => LogStash::Outputs::ElasticSearch::LicenseChecker.new(logger))
+    end
+
+    before(:each) do
+      allow(subject).to receive(:health_check_request)
+    end
+
     context "when using default logstash distribution" do
       let(:oss) { false }
+
       context "if ES doesn't return a valid license" do
         let(:valid_license) { false }
+
         it "marks the url as active" do
           subject.update_initial_urls
           expect(subject.alive_urls_count).to eq(1)
         end
+
         it "logs a warning" do
-          expect(subject).to receive(:log_license_deprecation_warn).once
+          expect(subject.license_checker).to receive(:log_license_deprecation_warn).once
           subject.update_initial_urls
         end
       end
+
       context "if ES returns a valid license" do
         let(:valid_license) { true }
+
         it "marks the url as active" do
           subject.update_initial_urls
           expect(subject.alive_urls_count).to eq(1)
         end
+
         it "does not log a warning" do
-          expect(subject).to_not receive(:log_license_deprecation_warn)
+          expect(subject.license_checker).to_not receive(:log_license_deprecation_warn)
           subject.update_initial_urls
         end
       end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -1,3 +1,4 @@
+require_relative "../../../../spec/es_spec_helper"
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
 require "java"
@@ -138,6 +139,27 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     end
   end
 
+  describe "index template" do
+    subject { described_class.new(base_options) }
+    let(:template_name) { "logstash" }
+    let(:template) { {} }
+    let(:get_response) {
+      double("response", :body => {})
+    }
+
+    it "should call composable index template in version 8+" do
+      expect(subject).to receive(:maximum_seen_major_version).and_return(8)
+      expect(subject.pool).to receive(:put).with("_index_template/#{template_name}", nil, anything).and_return(get_response)
+      subject.template_put(template_name, template)
+    end
+
+    it "should call index template in version < 8" do
+      expect(subject).to receive(:maximum_seen_major_version).and_return(7)
+      expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
+      subject.template_put(template_name, template)
+    end
+  end
+
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -28,4 +28,35 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
       expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/elasticsearch-7x.json")
     end
   end
+
+  describe "index template with ilm settings" do
+    let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
+    let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+    describe "in version 8+" do
+      let(:file_path) { described_class.default_template_path(8) }
+      let(:template) { described_class.read_template_file(file_path)}
+
+      it "should update settings" do
+        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+        described_class.add_ilm_settings_to_template(plugin, template)
+        expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
+        expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+        expect(template.include?('settings')).to be_falsey
+      end
+    end
+
+    describe "in version < 8" do
+      let(:file_path) { described_class.default_template_path(7) }
+      let(:template) { described_class.read_template_file(file_path)}
+
+      it "should update settings" do
+        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+        described_class.add_ilm_settings_to_template(plugin, template)
+        expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
+        expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+        expect(template.include?('template')).to be_falsey
+      end
+    end
+  end
 end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -355,6 +355,28 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "the pipeline option" do
+    context "with a sprintf and set pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+
+      let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
+
+      it "should interpolate the pipeline value and set it" do
+        expect(subject.event_action_tuple(event)[1]).to include(:pipeline => "my-ingest-pipeline")
+      end
+    end
+
+    context "with a sprintf and empty pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+
+      let(:event) { LogStash::Event.new("pipeline" => "") }
+
+      it "should interpolate the pipeline value but not set it because it is empty" do
+        expect(subject.event_action_tuple(event)[1]).not_to include(:pipeline)
+      end
+    end
+  end
+
   describe "SSL end to end" do
     let(:do_register) { false } # skip the register in the global before block, as is called here.
 

data/spec/unit/outputs/license_check_spec.rb

@@ -0,0 +1,41 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "logstash/outputs/elasticsearch/license_checker"
+
+describe LogStash::Outputs::ElasticSearch::LicenseChecker do
+
+  # Note that the actual license checking logic is spec'ed in pool_spec.rb
+
+  context "LicenseChecker API required by Pool class" do
+    subject { described_class }
+
+    it "defines the appropriate_license? methods" do
+      expect(subject.instance_methods).to include(:appropriate_license?)
+    end
+  end
+
+  context "LicenseChecker API required by Pool specs" do
+    subject { described_class }
+
+    it "defines the oss? method" do
+      expect(subject.instance_methods).to include(:oss?)
+    end
+
+    it "defines the valid_es_license? method" do
+      expect(subject.instance_methods).to include(:valid_es_license?)
+    end
+
+    it "defines the log_license_deprecation_warn method" do
+      expect(subject.instance_methods).to include(:log_license_deprecation_warn)
+    end
+  end
+
+  context "Pool class API required by the LicenseChecker" do
+    subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool }
+
+    it "contains the get_license method" do
+      expect(LogStash::Outputs::ElasticSearch::HttpClient::Pool.instance_methods).to include(:get_license)
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 10.6.1
+  version: 10.8.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2020-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +170,13 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/outputs/elasticsearch.rb
-- lib/logstash/outputs/elasticsearch/common.rb
-- lib/logstash/outputs/elasticsearch/common_configs.rb
 - lib/logstash/outputs/elasticsearch/default-ilm-policy.json
 - lib/logstash/outputs/elasticsearch/http_client.rb
 - lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb
 - lib/logstash/outputs/elasticsearch/http_client/pool.rb
 - lib/logstash/outputs/elasticsearch/http_client_builder.rb
 - lib/logstash/outputs/elasticsearch/ilm.rb
+- lib/logstash/outputs/elasticsearch/license_checker.rb
 - lib/logstash/outputs/elasticsearch/template_manager.rb
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json
@@ -186,6 +185,9 @@ files:
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
+- lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
+- lib/logstash/plugin_mixins/elasticsearch/common.rb
+- lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
 - spec/fixtures/_nodes/2x_1x.json
@@ -201,6 +203,7 @@ files:
 - spec/fixtures/scripts/painless/scripted_upsert.painless
 - spec/fixtures/template-with-policy-es6x.json
 - spec/fixtures/template-with-policy-es7x.json
+- spec/fixtures/template-with-policy-es8x.json
 - spec/fixtures/test_certs/ca/ca.crt
 - spec/fixtures/test_certs/ca/ca.key
 - spec/fixtures/test_certs/test.crt
@@ -237,6 +240,7 @@ files:
 - spec/unit/outputs/elasticsearch_spec.rb
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
 - spec/unit/outputs/error_whitelist_spec.rb
+- spec/unit/outputs/license_check_spec.rb
 homepage: http://logstash.net/
 licenses:
 - apache-2.0
@@ -278,6 +282,7 @@ test_files:
 - spec/fixtures/scripts/painless/scripted_upsert.painless
 - spec/fixtures/template-with-policy-es6x.json
 - spec/fixtures/template-with-policy-es7x.json
+- spec/fixtures/template-with-policy-es8x.json
 - spec/fixtures/test_certs/ca/ca.crt
 - spec/fixtures/test_certs/ca/ca.key
 - spec/fixtures/test_certs/test.crt
@@ -314,3 +319,4 @@ test_files:
 - spec/unit/outputs/elasticsearch_spec.rb
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
 - spec/unit/outputs/error_whitelist_spec.rb
+- spec/unit/outputs/license_check_spec.rb

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -1,167 +0,0 @@
-require 'forwardable' # Needed for logstash core SafeURI. We need to patch this in core: https://github.com/elastic/logstash/pull/5978
-
-module LogStash; module Outputs; class ElasticSearch
-  module CommonConfigs
-
-    DEFAULT_INDEX_NAME = "logstash-%{+yyyy.MM.dd}"
-    DEFAULT_POLICY = "logstash-policy"
-    DEFAULT_ROLLOVER_ALIAS = 'logstash'
-
-    DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//127.0.0.1")
-
-    def self.included(mod)
-      # The index to write events to. This can be dynamic using the `%{foo}` syntax.
-      # The default value will partition your indices by day so you can more easily
-      # delete old data or only search specific date ranges.
-      # Indexes may not contain uppercase characters.
-      # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
-      # LS uses Joda to format the index pattern from event timestamp.
-      # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
-      mod.config :index, :validate => :string
-
-      mod.config :document_type, 
-        :validate => :string, 
-        :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
-
-      # From Logstash 1.3 onwards, a template is applied to Elasticsearch during
-      # Logstash's startup if one with the name `template_name` does not already exist.
-      # By default, the contents of this template is the default template for
-      # `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
-      # `logstash-*`.  Should you require support for other index names, or would like
-      # to change the mappings in the template in general, a custom template can be
-      # specified by setting `template` to the path of a template file.
-      #
-      # Setting `manage_template` to false disables this feature.  If you require more
-      # control over template creation, (e.g. creating indices dynamically based on
-      # field names) you should set `manage_template` to false and use the REST
-      # API to apply your templates manually.
-      mod.config :manage_template, :validate => :boolean, :default => true
-
-      # This configuration option defines how the template is named inside Elasticsearch.
-      # Note that if you have used the template management features and subsequently
-      # change this, you will need to prune the old template manually, e.g.
-      #
-      # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
-      #
-      # where `OldTemplateName` is whatever the former setting was.
-      mod.config :template_name, :validate => :string
-
-      # You can set the path to your own template here, if you so desire.
-      # If not set, the included template will be used.
-      mod.config :template, :validate => :path
-
-      # The template_overwrite option will always overwrite the indicated template
-      # in Elasticsearch with either the one indicated by template or the included one.
-      # This option is set to false by default. If you always want to stay up to date
-      # with the template provided by Logstash, this option could be very useful to you.
-      # Likewise, if you have your own template file managed by puppet, for example, and
-      # you wanted to be able to update it regularly, this option could help there as well.
-      #
-      # Please note that if you are using your own customized version of the Logstash
-      # template (logstash), setting this to true will make Logstash to overwrite
-      # the "logstash" template (i.e. removing all customized settings)
-      mod.config :template_overwrite, :validate => :boolean, :default => false
-
-      # The document ID for the index. Useful for overwriting existing entries in
-      # Elasticsearch with the same ID.
-      mod.config :document_id, :validate => :string
-
-      # The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
-      # See https://www.elastic.co/blog/elasticsearch-versioning-support.
-      mod.config :version, :validate => :string
-      
-      # The version_type to use for indexing.
-      # See https://www.elastic.co/blog/elasticsearch-versioning-support.
-      # See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#_version_types
-      mod.config :version_type, :validate => ["internal", 'external', "external_gt", "external_gte", "force"]
-
-      # A routing override to be applied to all processed events.
-      # This can be dynamic using the `%{foo}` syntax.
-      mod.config :routing, :validate => :string
-
-      # For child documents, ID of the associated parent.
-      # This can be dynamic using the `%{foo}` syntax.
-      mod.config :parent, :validate => :string, :default => nil
-
-      # For child documents, name of the join field
-      mod.config :join_field, :validate => :string, :default => nil
-
-      # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
-      # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
-      #     `"127.0.0.1"`
-      #     `["127.0.0.1:9200","127.0.0.2:9200"]`
-      #     `["http://127.0.0.1"]`
-      #     `["https://127.0.0.1:9200"]`
-      #     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
-      # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
-      # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
-      #
-      # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
-      mod.config :hosts, :validate => :uri, :default => [ DEFAULT_HOST ], :list => true
-
-      # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
-      #
-      # For more details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[cloud documentation]
-      mod.config :cloud_id, :validate => :string
-
-      # Set upsert content for update mode.s
-      # Create a new document with this parameter as json string if `document_id` doesn't exists
-      mod.config :upsert, :validate => :string, :default => ""
-
-      # Enable `doc_as_upsert` for update mode.
-      # Create a new document with source if `document_id` doesn't exist in Elasticsearch
-      mod.config :doc_as_upsert, :validate => :boolean, :default => false
-
-      # Set script name for scripted update mode
-      mod.config :script, :validate => :string, :default => ""
-
-      # Define the type of script referenced by "script" variable
-      #  inline : "script" contains inline script
-      #  indexed : "script" contains the name of script directly indexed in elasticsearch
-      #  file    : "script" contains the name of script stored in elasticseach's config directory
-      mod.config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"]
-
-      # Set the language of the used script. If not set, this defaults to painless in ES 5.0
-      mod.config :script_lang, :validate => :string, :default => "painless"
-
-      # Set variable name passed to script (scripted update)
-      mod.config :script_var_name, :validate => :string, :default => "event"
-
-      # if enabled, script is in charge of creating non-existent document (scripted update)
-      mod.config :scripted_upsert, :validate => :boolean, :default => false
-
-      # Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
-      mod.config :retry_initial_interval, :validate => :number, :default => 2
-
-      # Set max interval in seconds between bulk retries.
-      mod.config :retry_max_interval, :validate => :number, :default => 64
-
-      # The number of times Elasticsearch should internally retry an update/upserted document
-      # See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
-      # for more info
-      mod.config :retry_on_conflict, :validate => :number, :default => 1
-
-      # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
-      # here like `pipeline => "%{INGEST_PIPELINE}"`
-      mod.config :pipeline, :validate => :string, :default => nil
-
-
-      # -----
-      # ILM configurations (beta)
-      # -----
-      # Flag for enabling Index Lifecycle Management integration.
-      mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto'
-
-      # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
-      mod.config :ilm_rollover_alias, :validate => :string
-
-      # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
-      # {now/d} is date math, and will insert the appropriate value automatically.
-      mod.config :ilm_pattern, :validate => :string, :default => '{now/d}-000001'
-
-      # ILM policy to use, if undefined the default policy will be used.
-      mod.config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
-
-    end
-  end
-end end end