logstash-filter-elasticsearch 3.7.1 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3546d5b2082aa7d3f83c0a855c6fcebe9203c83bb6d5a7831da2237c9b5b2672
-  data.tar.gz: fa20e8e10194ebbe86b3ceea46f582d4a0c50c66122bd75968a4df149d848648
+  metadata.gz: 77145aca4fea09d207fbe7efb17c271760fb7fd7eaf3c1411c08298959a54767
+  data.tar.gz: 599f7a6cfdc37a5f7f867b44411f82438a53a17fb8ed9bbbfe8c34f4aa9496d2
 SHA512:
-  metadata.gz: a4278733f509ed60089bcaae0e2d509acb66fd8b0ba3bb361d889b167991625e790a3d2422870a114ad3527f418ed2389b95fc7abee7334e8c2e7e07eb1dedc2
-  data.tar.gz: fffc8107d0ce39d3cc3d12669a0c13340f70fa5d1ec6807ea953b0f2d1f2049594372c6360ec87e7a8c2eb4b0682e85d0b906c6028e458ba7cf350ca45d5653d
+  metadata.gz: 6c764d630938fd180399b4d0f113bb3af17b5db41594838d6e4b109b280457443c97920999c74f62c0518bc69780bc8c8569383477e576dc80e6433ccbab3c02
+  data.tar.gz: 998fbdba7ede11d6ab4ab74c5620076f730490cb37831041fa947540917d7da519f9ff0b75989445f5a1a25937dc5e04fc7a33afa56ebec6cc8234d48b7e9444

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.8.0
+  - Added api_key support [#132](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/132)
+
+## 3.7.2
+  - [DOC] Removed outdated compatibility notice [#131](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/131)
+
 ## 3.7.1
   - Fix: solves an issue where non-ascii unicode values in a template were not handled correctly [#128](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/128)
 

data/CONTRIBUTORS

@@ -9,6 +9,7 @@ Contributors:
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
 * Adrian Solom (addrians)
+* Colin Surprenant (colinsurprenant)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/docs/index.asciidoc

@@ -20,18 +20,8 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-.Compatibility Note
-[NOTE]
-================================================================================
-Starting with Elasticsearch 5.3, there's an {ref}/modules-http.html[HTTP setting]
-called `http.content_type.required`. If this option is set to `true`, and you
-are using Logstash 2.4 through 5.2, you need to update the Elasticsearch filter
-plugin to version 3.1.1 or higher.
-
-================================================================================
-
 Search Elasticsearch for a previous log event and copy some fields from it
-into the current event.  Below are two complete examples of how this filter might
+into the current event. Below are two complete examples of how this filter might
 be used.
 
 The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
@@ -111,6 +101,14 @@ Notice also that when you use `query_template`, the Logstash attributes `result_
 and `sort` will be ignored. They should be specified directly in the JSON
 template, as shown in the example above.
 
+[id="plugins-{type}s-{plugin}-auth"]
+==== Authentication
+
+Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options:
+
+* <<plugins-{type}s-{plugin}-user>> AND <<plugins-{type}s-{plugin}-password>>
+* <<plugins-{type}s-{plugin}-cloud_auth>>
+* <<plugins-{type}s-{plugin}-api_key>>
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Filter Configuration Options
@@ -121,6 +119,7 @@ This plugin supports the following configuration options plus the <
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-aggregation_fields>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
@@ -162,6 +161,16 @@ Example:
       }
     }
 
+[id="plugins-{type}s-{plugin}-api_key"]
+===== `api_key`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option.
+
+Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key API].
+
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
 

data/lib/logstash/filters/elasticsearch.rb

@@ -15,11 +15,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # List of elasticsearch hosts to use for querying.
   config :hosts, :validate => :array, :default => [ DEFAULT_HOST ]
 
-  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
-  #
-  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
-  config :cloud_id, :validate => :string
-
   # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
   # Field substitution (e.g. `index-name-%{date_field}`) is available
   config :index, :validate => :string, :default => ""
@@ -50,11 +45,20 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+  config :cloud_id, :validate => :string
+
   # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
   #
   # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
   config :cloud_auth, :validate => :password
 
+  # Authenticate using Elasticsearch API key.
+  # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+  config :api_key, :validate => :password
+
   # SSL
   config :ssl, :validate => :boolean, :default => false
 
@@ -84,8 +88,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       @query_dsl = file.read
     end
 
-    fill_hosts_from_cloud_id
+    validate_authentication
     fill_user_password_from_cloud_auth
+    fill_hosts_from_cloud_id
 
     @hosts = Array(@hosts).map { |host| host.to_s } # for ES client URI#to_s
 
@@ -156,19 +161,21 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end # def filter
 
   private
+
   def client_options
     {
+      :user => @user,
+      :password => @password,
+      :api_key => @api_key,
       :ssl => @ssl,
-      :hosts => @hosts,
       :ca_file => @ca_file,
-      :logger => @logger
     }
   end
 
   def new_client
     # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
     # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
-    LogStash::Filters::ElasticsearchClient.new(@user, @password, client_options)
+    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, client_options)
   end
 
   def get_client
@@ -213,25 +220,37 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(DEFAULT_HOST)
   end
 
-  def fill_hosts_from_cloud_id
-    return unless @cloud_id
+  def validate_authentication
+    authn_options = 0
+    authn_options += 1 if @cloud_auth
+    authn_options += 1 if (@api_key && @api_key.value)
+    authn_options += 1 if (@user || (@password && @password.value))
 
-    if @hosts && !hosts_default?(@hosts)
-      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    if authn_options > 1
+      raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
+    end
+
+    if @api_key && @api_key.value && @ssl != true
+      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
     end
-    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
   end
 
   def fill_user_password_from_cloud_auth
     return unless @cloud_auth
 
-    if @user || @password
-      raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
-    end
     @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
     params['user'], params['password'] = @user, @password
   end
 
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
   def parse_host_uri_from_cloud_id(cloud_id)
     begin # might not be available on older LS
       require 'logstash/util/cloud_setting_id'

data/lib/logstash/filters/elasticsearch/client.rb

@@ -10,23 +10,22 @@ module LogStash
 
       attr_reader :client
 
-      def initialize(user, password, options={})
-        ssl     = options.fetch(:ssl, false)
-        hosts   = options[:hosts]
-        @logger = options[:logger]
+      def initialize(logger, hosts, options = {})
+        ssl = options.fetch(:ssl, false)
+        user = options.fetch(:user, nil)
+        password = options.fetch(:password, nil)
+        api_key = options.fetch(:api_key, nil)
 
-        transport_options = {}
-        if user && password
-          token = ::Base64.strict_encode64("#{user}:#{password.value}")
-          transport_options[:headers] = { Authorization: "Basic #{token}" }
-        end
+        transport_options = {:headers => {}}
+        transport_options[:headers].merge!(setup_basic_auth(user, password))
+        transport_options[:headers].merge!(setup_api_key(api_key))
 
         hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
         # set ca_file even if ssl isn't on, since the host can be an https url
         ssl_options = { ssl: true, ca_file: options[:ca_file] } if options[:ca_file]
         ssl_options ||= {}
 
-        @logger.info("New ElasticSearch filter client", :hosts => hosts)
+        logger.info("New ElasticSearch filter client", :hosts => hosts)
         @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options)
       end
 
@@ -34,6 +33,21 @@ module LogStash
         @client.search(params)
       end
 
+      private
+
+      def setup_basic_auth(user, password)
+        return {} unless user && password && password.value
+
+        token = ::Base64.strict_encode64("#{user}:#{password.value}")
+        { Authorization: "Basic #{token}" }
+      end
+
+      def setup_api_key(api_key)
+        return {} unless (api_key && api_key.value)
+
+        token = ::Base64.strict_encode64(api_key.value)
+        { Authorization: "ApiKey #{token}" }
+      end
     end
   end
 end

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.7.1'
+  s.version         = '3.8.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -367,7 +367,37 @@ describe LogStash::Filters::Elasticsearch do
         let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
         it "should fail" do
-          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "api_key" do
+      context "without ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
+        end
+      end
+
+      context "with ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+
+        it "should set authorization" do
+          plugin.register
+          client = plugin.send(:get_client).client
+          auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
+        end
+
+        context 'user also set' do
+          let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+
+          it "should fail" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+          end
         end
       end
     end if LOGSTASH_VERSION > '6.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.7.1
+  version: 3.8.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-09 00:00:00.000000000 Z
+date: 2020-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement