logstash-filter-elasticsearch 3.6.1 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54415da2d9bc4aa6d1d90ea0b65937aa4782c9a9614cafd2ac583d94c0780913
-  data.tar.gz: 44eea0bc6efd2a0a2a5415320a872e03cd3883e122e8d7627c3873241d47720c
+  metadata.gz: f8fd49609af063aa843c5af56f09194753911fab6bc057dc8bde3c4e84c196b1
+  data.tar.gz: 728f386a7e634276b02705e2c37ad9316d8f490cf3811c3e85f7a0203485e480
 SHA512:
-  metadata.gz: f148fcacc216cbbfca630e858ec93d85329487738dda440200ac2ea4f85253f2f4b6dc4a55cb859781be372a7ef15c727b5ded25015de1988302cd04e64de98e
-  data.tar.gz: '003910ce329cf1945fb3f92c2aef9f096588eb33c85e8716dbba0b65dd770e168e3960b1751553c2d251bc43b2a36dbc3eef339720acc9d40a08e1d1d36db402'
+  metadata.gz: e4a567ac1f292ed3aced93934029b1048818c7ec930d88e86a3bffaa9dec27d90f54ba70b885771804fcb7175370ee67907de206d464af8e8658f6e4649f6607
+  data.tar.gz: bde67c1c10b2e1fcdde0160794d565389d443fb0667e45a3a5363d3f67f6e00571b7f284bbdc560721560965e1b5194c4e7087e345e838216b961c027e46f5e6

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.7.0
+  - Feat: support cloud_id / cloud_auth configuration [#122](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/122)
+
 ## 3.6.1
   - Loosen restrictions on Elasticsearch gem ([#120](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/120))
 

data/docs/index.asciidoc

@@ -122,6 +122,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-aggregation_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-enable_sort>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
@@ -299,6 +301,26 @@ Tags the event on failure to look up previous log event information. This can be
 
 Basic Auth - username
 
+[id="plugins-{type}s-{plugin}-cloud_auth"]
+===== `cloud_auth`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
+
+For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+
+[id="plugins-{type}s-{plugin}-cloud_id"]
+===== `cloud_id`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+
+For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+
 
 
 [id="plugins-{type}s-{plugin}-common-options"]

data/lib/logstash/filters/elasticsearch.rb

@@ -3,15 +3,23 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require_relative "elasticsearch/client"
 require "logstash/json"
+require "logstash/util/safe_uri"
 java_import "java.util.concurrent.ConcurrentHashMap"
 
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config_name "elasticsearch"
 
+  DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//localhost:9200")
+
   # List of elasticsearch hosts to use for querying.
-  config :hosts, :validate => :array,  :default => [ "localhost:9200" ]
-  
+  config :hosts, :validate => :array, :default => [ DEFAULT_HOST ]
+
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[cloud documentation]
+  config :cloud_id, :validate => :string
+
   # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
   # Field substitution (e.g. `index-name-%{date_field}`) is available
   config :index, :validate => :string, :default => ""
@@ -42,6 +50,11 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[cloud documentation]
+  config :cloud_auth, :validate => :password
+
   # SSL
   config :ssl, :validate => :boolean, :default => false
 
@@ -71,6 +84,11 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       @query_dsl = file.read
     end
 
+    fill_hosts_from_cloud_id
+    fill_user_password_from_cloud_auth
+
+    @hosts = Array(@hosts).map { |host| host.to_s } # for ES client URI#to_s
+
     test_connection!
   end # def register
 
@@ -148,6 +166,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def new_client
+    # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
+    # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
     LogStash::Filters::ElasticsearchClient.new(@user, @password, client_options)
   end
 
@@ -188,6 +208,64 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     total
   end
 
+  def hosts_default?(hosts)
+    # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
+    hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(DEFAULT_HOST)
+  end
+
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
+  def fill_user_password_from_cloud_auth
+    return unless @cloud_auth
+
+    if @user || @password
+      raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
+    end
+    @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
+    params['user'], params['password'] = @user, @password
+  end
+
+  def parse_host_uri_from_cloud_id(cloud_id)
+    begin # might not be available on older LS
+      require 'logstash/util/cloud_setting_id'
+    rescue LoadError
+      raise LogStash::ConfigurationError, 'The cloud_id setting is not supported by your version of Logstash, ' +
+          'please upgrade your installation (or set hosts instead).'
+    end
+
+    begin
+      cloud_id = LogStash::Util::CloudSettingId.new(cloud_id) # already does append ':{port}' to host
+    rescue ArgumentError => e
+      raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Id/i, 'cloud_id')
+    end
+    cloud_uri = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
+    LogStash::Util::SafeURI.new(cloud_uri)
+  end
+
+  def parse_user_password_from_cloud_auth(cloud_auth)
+    begin # might not be available on older LS
+      require 'logstash/util/cloud_setting_auth'
+    rescue LoadError
+      raise LogStash::ConfigurationError, 'The cloud_auth setting is not supported by your version of Logstash, ' +
+          'please upgrade your installation (or set user/password instead).'
+    end
+
+    cloud_auth = cloud_auth.value if cloud_auth.is_a?(LogStash::Util::Password)
+    begin
+      cloud_auth = LogStash::Util::CloudSettingAuth.new(cloud_auth)
+    rescue ArgumentError => e
+      raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Auth/i, 'cloud_auth')
+    end
+    [ cloud_auth.username, cloud_auth.password ]
+  end
+
   def test_connection!
     get_client.client.ping
   end

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.6.1'
+  s.version         = '3.7.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -289,7 +289,87 @@ describe LogStash::Filters::Elasticsearch do
       end
 
     end
-
   end
 
+  describe "client" do
+    let(:config) do
+      {
+          "query" => "response: unknown"
+      }
+    end
+    let(:plugin) { described_class.new(config) }
+    let(:event)  { LogStash::Event.new({}) }
+
+    before(:each) do
+      allow(plugin).to receive(:test_connection!)
+    end
+
+    after(:each) do
+      Thread.current[:filter_elasticsearch_client] = nil
+    end
+
+    describe "cloud.id" do
+      let(:valid_cloud_id) do
+        'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
+      end
+
+      let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
+
+      it "should set host(s)" do
+        plugin.register
+        client = plugin.send(:get_client).client
+        expect( client.transport.hosts ).to eql [{
+                                                     :scheme => "https",
+                                                     :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
+                                                     :port => 9243,
+                                                     :path => "",
+                                                     :protocol => "https"
+                                                 }]
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
+        end
+      end
+
+      context 'hosts also set' do
+        let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "cloud.auth" do
+      let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
+
+      it "should set authorization" do
+        plugin.register
+        client = plugin.send(:get_client).client
+        auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+        expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
+        end
+      end
+
+      context 'user also set' do
+        let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.6.1
+  version: 3.7.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-12 00:00:00.000000000 Z
+date: 2020-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement