logstash-output-elasticsearch 11.4.1-java → 11.5.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1ce0504aa5082dc3fc958b72019f243c6cad0174bb883612c25eb5d7f8aedfe
-  data.tar.gz: 4d28c34d89b5a70a02c10ebd3b8d425111d3902bc58fb9e428acadee4a471646
+  metadata.gz: 7eab2dc342c636e2d5df4e2f9bbd776b446cf110a0803c11fc919fcbbd2f2d83
+  data.tar.gz: 565ed3031b685f8541853043d486bda0cff8aa7da2c6f2733e0e2b43345de099
 SHA512:
-  metadata.gz: d1d0680cf639e01b6c2a2a3b8eb87a2ad525764d006fb3cbfadc06c845925d9f1055075265d7327794e326b1a9df67308d85ebefafb38297bd2121499ecc2279
-  data.tar.gz: 285bafc2ffa2ae726bdafdb3176c193c7cd20979625e59cd7526445ae5243137e6dad37cc64208bb3494fc45afe44d913f7b7b4188fcf6b25faf1e9fb6a04d06
+  metadata.gz: 1fbb452170d276531d9b202538d9cf0a23b834053309c818aaaff6f75dd41ecd3d787394234c295639d95d2fe763d1063424cff56b598d1ba16e2fe92374df44
+  data.tar.gz: 6b7f17fa679306e65ec94a048ef2205ec9f884cbd11fe1adf861dbb1eccaea752105c3bf149a4fe08f1cdb3c6189200d612dc4cece52ac937eb78d599b188926

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.5.0
+ - Feat: add ssl_supported_protocols option [#1055](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1055)
+
+## 11.4.2
+ - [DOC] Add `v8` to supported values for ecs_compatiblity defaults [#1059](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1059)
+
 ## 11.4.1
  - Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063)
    - the underlying changes include latest HttpClient (4.5.13)

data/docs/index.asciidoc

@@ -355,6 +355,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-sniffing_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
@@ -554,7 +555,7 @@ If you don't set a value for this option:
 * Value type is <<string,string>>
 * Supported values are:
 ** `disabled`: does not provide ECS-compatible templates
-** `v1`: provides defaults that are compatible with v1 of the Elastic Common Schema
+** `v1`,`v8`: Elastic Common Schema-compliant behavior
 * Default value depends on which version of Logstash is running:
 ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
 ** Otherwise, the default value is `disabled`.
@@ -1004,6 +1005,23 @@ Option to validate the server's certificate. Disabling this severely compromises
 For more information on disabling certificate verification please read
 https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
 
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+    `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
+
+List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
+
+For Java 8 `'TLSv1.3'` is supported  only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
+
 [id="plugins-{type}s-{plugin}-template"]
 ===== `template`
 
@@ -1018,8 +1036,8 @@ If not set, the included template will be used.
 
   * Value type is <<string,string>>
   * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
-  ** ECS Compatibility disabled: `logstash`
-  ** ECS Compatibility enabled: `ecs-logstash`
+    ** ECS Compatibility disabled: `logstash`
+    ** ECS Compatibility enabled: `ecs-logstash`
 
 
 This configuration option defines how the template is named inside Elasticsearch.

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -283,11 +283,11 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def client_settings
-      @options[:client_settings] || {}
+      @_client_settings ||= @options[:client_settings] || {}
     end
 
     def ssl_options
-      client_settings.fetch(:ssl, {})
+      @_ssl_options ||= client_settings.fetch(:ssl, {})
     end
 
     def http_compression

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -132,11 +132,16 @@ module LogStash; module Outputs; class ElasticSearch;
         ssl_options[:keystore] = keystore
         ssl_options[:keystore_password] = keystore_password.value if keystore_password
       end
+
       if !params["ssl_certificate_verification"]
         logger.warn "You have enabled encryption but DISABLED certificate verification, " +
                     "to make sure your data is secure remove `ssl_certificate_verification => false`"
         ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
       end
+
+      protocols = params['ssl_supported_protocols']
+      ssl_options[:protocols] = protocols if protocols && protocols.any?
+
       { ssl: ssl_options }
     end
 

data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json

@@ -4,7 +4,7 @@
   ],
   "mappings": {
     "_meta": {
-      "version": "8.0.0-dev"
+      "version": "8.0.1"
     },
     "date_detection": false,
     "dynamic_templates": [

data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json

@@ -15,7 +15,7 @@
     },
     "mappings": {
       "_meta": {
-        "version": "8.0.0-dev"
+        "version": "8.0.1"
       },
       "date_detection": false,
       "dynamic_templates": [

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -66,6 +66,8 @@ module LogStash; module PluginMixins; module ElasticSearch
         # Set the keystore password
         :keystore_password => { :validate => :password },
 
+        :ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true },
+
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
         # this with master nodes, you probably want to disable HTTP on them by setting

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.4.1'
-
+  s.version         = '11.5.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/index_spec.rb

@@ -60,25 +60,48 @@ describe "indexing" do
 
   let(:curl_opts) { nil }
 
+  let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
+  let(:es_admin_pass) { 'elastic' }
+
   def curl_and_get_json_response(url, method: :get); require 'open3'
+    cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
     begin
-      stdout, status = Open3.capture2("curl #{curl_opts} -X #{method.to_s.upcase} -k #{url}")
+      out, err, status = Open3.capture3(cmd)
     rescue Errno::ENOENT
       fail "curl not available, make sure curl binary is installed and available on $PATH"
     end
 
     if status.success?
-      LogStash::Json.load(stdout)
+      http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+
+      if http_status.strip[0].to_i > 2
+        error = (LogStash::Json.load(out)['error']) rescue nil
+        if error
+          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
+        else
+          warn out
+          fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+        end
+      end
+
+      LogStash::Json.load(out)
     else
-      fail "curl failed: #{status}\n  #{stdout}"
+      warn out
+      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
     end
   end
 
+  let(:initial_events) { [] }
+
   before do
     subject.register
-    subject.multi_receive([])
+    subject.multi_receive(initial_events) if initial_events
   end
-  
+
+  after do
+    subject.do_close
+  end
+
   shared_examples "an indexer" do |secure|
     it "ships events" do
       subject.multi_receive(events)
@@ -146,17 +169,17 @@ describe "indexing" do
     let(:user) { "simpleuser" }
     let(:password) { "abc123" }
     let(:cacert) { "spec/fixtures/test_certs/ca.crt" }
-    let(:es_url) {"https://elasticsearch:9200"}
+    let(:es_url) { "https://#{get_host_port}" }
     let(:config) do
       {
-        "hosts" => ["elasticsearch:9200"],
+        "hosts" => [ get_host_port ],
         "user" => user,
         "password" => password,
         "ssl" => true,
         "cacert" => cacert,
         "index" => index
       }
-    end
+    end 
 
     let(:curl_opts) { "-u #{user}:#{password}" }
 
@@ -197,6 +220,8 @@ describe "indexing" do
 
     else
 
+      let(:curl_opts) { "#{super()} --tlsv1.2 --tls-max 1.3 -u #{es_admin}:#{es_admin_pass}" } # due ES 8.x we need user/password
+
       it_behaves_like("an indexer", true)
 
       describe "with a password requiring escaping" do
@@ -219,6 +244,32 @@ describe "indexing" do
         include_examples("an indexer", true)
       end
 
+      context 'with enforced TLSv1.3 protocol' do
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.3' ] }
+
+        it_behaves_like("an indexer", true)
+      end
+
+      context 'with enforced TLSv1.2 protocol (while ES only enabled TLSv1.3)' do
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.2' ] }
+
+        let(:initial_events) { nil }
+
+        it "does not ship events" do
+          curl_and_get_json_response index_url, method: :put # make sure index exists
+          Thread.start { subject.multi_receive(events) } # we'll be stuck in a retry loop
+          sleep 2.5
+
+          curl_and_get_json_response "#{es_url}/_refresh", method: :post
+
+          result = curl_and_get_json_response "#{index_url}/_count?q=*"
+          cur_count = result["count"]
+          expect(cur_count).to eq(0) # ES output keeps re-trying but ends up with a
+          # [Manticore::ClientProtocolException] Received fatal alert: protocol_version
+        end
+
+      end if ENV['ES_SSL_SUPPORTED_PROTOCOLS'] == 'TLSv1.3'
+
     end
 
   end

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -33,7 +33,7 @@ describe "SSL option" do
     
     it "should pass the flag to the ES client" do
       expect(::Manticore::Client).to receive(:new) do |args|
-        expect(args[:ssl]).to eq(:enabled => true, :verify => :disable)
+        expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :disable)
       end.and_return(manticore_double)
       
       subject.register

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.4.1
+  version: 11.5.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-19 00:00:00.000000000 Z
+date: 2022-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement