logstash-output-elasticsearch 10.2.3-java → 10.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6be61b215a17faa3cec976576510d0f153be3a343162fa921e77398659b764a7
-  data.tar.gz: 35ab3d0b82ada32bd091091b2f1759681b976356837301c35bf0d14016cc6fa9
+  metadata.gz: 95ca05594f4d76e1bb341f14088bcafab677339c4322db449c24022d320d22ee
+  data.tar.gz: 79ec3c6d83999b68f9ebfca2c144c5556badfb9046d1f5c968628319623001d4
 SHA512:
-  metadata.gz: 134a73bd275a3f1c50d453fc9d4ee3f7b17a54293099d0b0c539e8cb87f2fc66993f2ad237b62ea789580a18a0ddeb785bade259ca24305d0211cea18f477e52
-  data.tar.gz: 23f201b5a03917d066f558f34d3b14d7bb6218cc8ef090ec3d26766f4aaab318d430256463a61cc00828fc60a3f24de290411a0015dbd689dd93a9b8add4bf87
+  metadata.gz: '049db8893d70c206be9cfe8e3e93354c98bfbd50d0718b936db79f6e9a07a3927fcdacc352f6e0eec02c0ac22ebd5cc866cd8193efa18a93a9c87f1a40251e7e'
+  data.tar.gz: 6509ec1cc74743832955ed52ecd868546f64712d99f6984285390b41adb71dd09d179d3e180decf9596e587f1b374cf611d0bda33e1e93b8421625d439fd8ae1

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 10.3.0
+  - Feat: Added support for cloud_id and cloud_auth [#906](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/906)
+
 ## 10.2.3
   - Opened type removal logic for extension. This allows X-Pack Elasticsearch output to continue using types for special case `/_monitoring` bulk endpoint, enabling a fix for LogStash #11312. [#900](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/900)
 

data/docs/index.asciidoc

@@ -230,6 +230,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-action>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-bulk_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
@@ -325,6 +327,26 @@ this defaults to a concatenation of the path parameter and "_bulk"
 
 The .cer or .pem file to validate the server's certificate
 
+[id="plugins-{type}s-{plugin}-cloud_auth"]
+===== `cloud_auth`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
+
+For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[cloud documentation]
+
+[id="plugins-{type}s-{plugin}-cloud_id"]
+===== `cloud_id`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+
+For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[cloud documentation]
+
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert` 
 
@@ -340,8 +362,7 @@ Create a new document with source if `document_id` doesn't exist in Elasticsearc
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-The document ID for the index. Useful for overwriting existing entries in
-Elasticsearch with the same ID.
+The document ID for the index. Useful for overwriting existing entries in Elasticsearch with the same ID.
 
 [id="plugins-{type}s-{plugin}-document_type"]
 ===== `document_type` 

data/lib/logstash/outputs/elasticsearch.rb

@@ -17,7 +17,7 @@ require "forwardable"
 # called `http.content_type.required`. If this option is set to `true`, and you
 # are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
 # plugin to version 6.2.5 or higher.
-# 
+#
 # ================================================================================
 #
 # This plugin is the recommended method of storing logs in Elasticsearch.
@@ -26,8 +26,8 @@ require "forwardable"
 # This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
 # We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
 # yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
-# to upgrade Logstash in lock-step. 
-# 
+# to upgrade Logstash in lock-step.
+#
 # You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
 #
 # ==== Template management for Elasticsearch 5.x
@@ -122,6 +122,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Password to authenticate to a secure Elasticsearch cluster
   config :password, :validate => :password
 
+  # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
+  #
+  # For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[cloud documentation]
+  config :cloud_auth, :validate => :password
+
   # HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps
   # the root path for the Elasticsearch HTTP API lives.
   # Note that if you use paths as components of URLs in the 'hosts' field you may

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -21,6 +21,8 @@ module LogStash; module Outputs; class ElasticSearch;
       # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
       @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
+      fill_hosts_from_cloud_id
+      fill_user_password_from_cloud_auth
       setup_hosts # properly sets @hosts
       build_client
       setup_after_successful_connection
@@ -118,6 +120,67 @@ module LogStash; module Outputs; class ElasticSearch;
       end
     end
 
+    def hosts_default?(hosts)
+      # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
+      hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(CommonConfigs::DEFAULT_HOST)
+    end
+    private :hosts_default?
+
+    def fill_hosts_from_cloud_id
+      return unless @cloud_id
+
+      if @hosts && !hosts_default?(@hosts)
+        raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+      end
+      @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+    end
+
+    def fill_user_password_from_cloud_auth
+      return unless @cloud_auth
+
+      if @user || @password
+        raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
+      end
+      @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
+      params['user'], params['password'] = @user, @password
+    end
+
+    def parse_host_uri_from_cloud_id(cloud_id)
+      begin # might not be available on older LS
+        require 'logstash/util/cloud_setting_id'
+      rescue LoadError
+        raise LogStash::ConfigurationError, 'The cloud_id setting is not supported by your version of Logstash, ' +
+            'please upgrade your installation (or set hosts instead).'
+      end
+
+      begin
+        cloud_id = LogStash::Util::CloudSettingId.new(cloud_id) # already does append ':{port}' to host
+      rescue ArgumentError => e
+        raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Id/i, 'cloud_id')
+      end
+      cloud_uri = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
+      LogStash::Util::SafeURI.new(cloud_uri)
+    end
+    private :parse_host_uri_from_cloud_id
+
+    def parse_user_password_from_cloud_auth(cloud_auth)
+      begin # might not be available on older LS
+        require 'logstash/util/cloud_setting_auth'
+      rescue LoadError
+        raise LogStash::ConfigurationError, 'The cloud_auth setting is not supported by your version of Logstash, ' +
+            'please upgrade your installation (or set user/password instead).'
+      end
+
+      cloud_auth = cloud_auth.value if cloud_auth.is_a?(LogStash::Util::Password)
+      begin
+        cloud_auth = LogStash::Util::CloudSettingAuth.new(cloud_auth)
+      rescue ArgumentError => e
+        raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Auth/i, 'cloud_auth')
+      end
+      [ cloud_auth.username, cloud_auth.password ]
+    end
+    private :parse_user_password_from_cloud_auth
+
     def maximum_seen_major_version
       client.maximum_seen_major_version
     end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -7,6 +7,8 @@ module LogStash; module Outputs; class ElasticSearch
     DEFAULT_POLICY = "logstash-policy"
     DEFAULT_ROLLOVER_ALIAS = 'logstash'
 
+    DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//127.0.0.1")
+
     def self.included(mod)
       # The index to write events to. This can be dynamic using the `%{foo}` syntax.
       # The default value will partition your indices by day so you can more easily
@@ -93,9 +95,14 @@ module LogStash; module Outputs; class ElasticSearch
       #     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
       # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
       # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
-      # 
+      #
       # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
-      mod.config :hosts, :validate => :uri, :default => [::LogStash::Util::SafeURI.new("//127.0.0.1")], :list => true
+      mod.config :hosts, :validate => :uri, :default => [ DEFAULT_HOST ], :list => true
+
+      # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+      #
+      # For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[cloud documentation]
+      mod.config :cloud_id, :validate => :string
 
       # Set upsert content for update mode.s
       # Create a new document with this parameter as json string if `document_id` doesn't exists

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.2.3'
+  s.version         = '10.3.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -337,18 +337,9 @@ describe LogStash::Outputs::ElasticSearch do
 
   describe "SSL end to end" do
     let(:do_register) { false } # skip the register in the global before block, as is called here.
-    let(:manticore_double) do
-      double("manticoreX#{self.inspect}")
-    end
 
     before(:each) do
-      response_double = double("manticore response").as_null_object
-      # Allow healtchecks
-      allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
-      allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
-      allow(manticore_double).to receive(:close)
-
-      allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
+      stub_manticore_client!
       subject.register
     end
 
@@ -522,6 +513,75 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "cloud.id" do
+    let(:do_register) { false }
+
+    let(:valid_cloud_id) do
+      'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
+    end
+
+    let(:options) { { 'cloud_id' => valid_cloud_id } }
+
+    before(:each) do
+      stub_manticore_client!
+    end
+
+    it "should set host(s)" do
+      subject.register
+      es_url = subject.client.pool.urls.first
+      expect( es_url.to_s ).to eql('https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243/')
+    end
+
+    context 'invalid' do
+      let(:options) { { 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' } }
+
+      it "should fail" do
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
+      end
+    end
+
+    context 'hosts also set' do
+      let(:options) { { 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost' ] } }
+
+      it "should fail" do
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
+      end
+    end
+  end
+
+  describe "cloud.auth" do
+    let(:do_register) { false }
+
+    let(:options) { { 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') } }
+
+    before(:each) do
+      stub_manticore_client!
+    end
+
+    it "should set host(s)" do
+      subject.register
+      es_url = subject.client.pool.urls.first
+      expect( es_url.user ).to eql('elastic')
+      expect( es_url.password ).to eql('my-passwd-00')
+    end
+
+    context 'invalid' do
+      let(:options) { { 'cloud_auth' => 'invalid-format' } }
+
+      it "should fail" do
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
+      end
+    end
+
+    context 'user also set' do
+      let(:options) { { 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' } }
+
+      it "should fail" do
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+      end
+    end
+  end
+
   context 'handling elasticsearch document-level status meant for the DLQ' do
     let(:options) { { "manage_template" => false } }
 
@@ -598,4 +658,18 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
   end
+
+  @private
+
+  def stub_manticore_client!(manticore_double = nil)
+    manticore_double ||= double("manticore #{self.inspect}")
+    response_double = double("manticore response").as_null_object
+    # Allow healtchecks
+    allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:close)
+
+    allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 10.2.3
+  version: 10.3.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-20 00:00:00.000000000 Z
+date: 2020-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement