logstash-input-tcp 6.0.8-java → 6.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 574feed9b3754fbb15bad08e4d670e9771261f1f90cbe45731fd3fd58b30deec
-  data.tar.gz: 259fc46a2885620249b23bd3d927a0a9948fc71544a7ea775569098e005da9e0
+  metadata.gz: 29b44274cfe25c623273407b63c29125143a6ec704c274732b73e9a3fc39f085
+  data.tar.gz: 4bced499147e26e9f8257d2c7a489a7dae93ec2a1c06faf2b45b3d9af801d222
 SHA512:
-  metadata.gz: 55d972be5de090da3e6142a54413064590495b2773b1c6bc21579e3e80ff644ba57b4a7a1083bf629fddd6e4ac85282ea74e5d9da4f05810b5cb45c50f176bfb
-  data.tar.gz: 662ca5182f7b4e4b6f40479c5a984eed7df2837c1853b1d42632270a239b162cfd490ec4d5079b59aa2cf1e6480bf3acf2e846fe268d8f3c836ea6d19daa66e5
+  metadata.gz: d45ef4273d94e4dc3dbcc94749d0e8d907ea96acb20ee4aeff94251a5390bb26b51105fc2f6104e09b885dc195f9fbe94261047035f905884d8306649e8aec49
+  data.tar.gz: f00ff1165183ce006a12541b7836bb3a09496eb4370bca185104ea81c8fc8e2360aae8ed09eeb170ab93ee9e64e7233af9360e4cbb73fc5d9b49adf27d26534b

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+## 6.2.0
+ - Added ECS Compatibility Mode [#165](https://github.com/logstash-plugins/logstash-input-tcp/pull/165)
+   - When operating in an ECS Compatibility mode, metadata about the connection on which we are receiving data is nested in well-named fields under `[@metadata][input][tcp]` instead of at the root level.
+ - Fix: source address is no longer missing when a proxy is present
+
+## 6.1.1
+ - Changed jar dependencies to reflect newer versions [#179](https://github.com/logstash-plugins/logstash-input-http/pull/179)
+
+## 6.1.0
+  - Feat: improve SSL error logging/unwrapping [#178](https://github.com/logstash-plugins/logstash-input-tcp/pull/178)
+  - Fix: the plugin will no longer have a side effect of adding the Bouncy-Castle security provider at runtime  
+
+## 6.0.10
+  - bumping dependency commons-io [#174](https://github.com/logstash-plugins/logstash-input-tcp/pull/174)
+
+## 6.0.9
+  - [DOC] Reorder options alphabetically [#171](https://github.com/logstash-plugins/logstash-input-tcp/pull/171)
+
 ## 6.0.8
   - [DOC] better description for `tcp_keep_alive` option [#169](https://github.com/logstash-plugins/logstash-input-tcp/pull/169)
 

data/docs/index.asciidoc

@@ -70,6 +70,52 @@ event timestamp
       }
     }
 
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Event Metadata and the Elastic Common Schema (ECS)
+
+In addition to decoding the events, this input will add metadata about the TCP connection itself to each event.
+This can be helpful when applications are configured to send events directly to this input's TCP listener without including information about themselves.
+
+Historically, this metadata was added to a variety of non-standard top-level fields, which had the potential to create confusion and schema conflicts downstream.
+With ECS compatibility mode, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace.
+
+[cols="3,7,5"]
+|=======================================================================
+| Metadata Group                                     | ecs: `v1`, `v8`                       | ecs: `disabled`
+
+.3+|Source Metadata from the TCP connection
+on which events are being received, including
+the sender's name, ip, and outbound port.       l|[@metadata][input][tcp][source][name] l|[host]
+l|[@metadata][input][tcp][source][ip]   l|[@metadata][ip_address]
+l|[@metadata][input][tcp][source][port] l|[port]
+
+.2+|Proxy Metadata from a proxied TCP connection.
+Available when receiving events by proxy and
+`proxy_protocol => true`                        l|[@metadata][input][tcp][proxy][ip]    l|[proxy_host]
+l|[@metadata][input][tcp][proxy][port]  l|[proxy_port]
+
+.1+|SSL Subject Metadata from a secured TCP
+connection. Available when `ssl_enable => true`
+AND `ssl_verify => true`                        l|[@metadata][input][tcp][ssl][subject] l|[sslsubject]
+|=======================================================================
+
+For example, the Elastic Common Schema reserves the https://www.elastic.co/guide/en/ecs/current/ecs-host.html[top-level `host` field] for information about the host on which the event happened.
+If an event is missing this metadata, it can be copied into place from the source TCP connection metadata that has been added to the event:
+
+[source,txt]
+-----
+filter {
+  if [@metadata][input][tcp][source] and not [host] {
+    mutate {
+      copy {
+        "[@metadata][input][tcp][source][name]" => "[host][name]"
+        "[@metadata][input][tcp][source][ip]"   => "[host][ip]"
+      }
+    }
+  }
+}
+-----
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Tcp Input Configuration Options
 
@@ -78,6 +124,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-mode>> |<<string,string>>, one of `["server", "client"]`|No
 | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
@@ -90,7 +138,6 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ssl_key_passphrase>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ssl_verify>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-tcp_keep_alive>> |<<boolean,boolean>>|No
-| <<plugins-{type}s-{plugin}-dns_reverse_lookup_enabled>> |<<boolean,boolean>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -98,6 +145,30 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"]
+===== `dns_reverse_lookup_enabled`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled,
+the address metadata that is added to events will contain the source address as-specified
+at the TCP layer and IPs will not be resolved to hostnames.
+
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+* Value type is <<string,string>>
+* Supported values are:
+** `disabled`: unstructured connection metadata added at root level
+** `v1`,`v8`: structured connection metadata added under `[@metadata][input][tcp]`
+* Default value depends on which version of Logstash is running:
+** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)].
+The value of this setting affects the <<plugins-{type}s-{plugin}-ecs_metadata,placement of a TCP connection's metadata>> on events.
+
 [id="plugins-{type}s-{plugin}-host"]
 ===== `host` 
 
@@ -206,16 +277,6 @@ Instruct the socket to use TCP keep alive. If it's `true` then the underlying so
 will use the OS defaults settings for keep alive. If it's `false` it doesn't configure any
 keep alive setting for the underlying socket.
 
-[id="plugins-{type}s-{plugin}-dns_reverse_lookup_enabled"]
-===== `dns_reverse_lookup_enabled`
-
-  * Value type is <<boolean,boolean>>
-  * Default value is `true`
-
-It is possible to avoid DNS reverse-lookups by disabling this setting. If disabled,
-the address metadata that is added to events will contain the source address as-specified
-at the TCP layer and IPs will not be resolved to hostnames.
-
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/tcp.rb

@@ -5,8 +5,7 @@ require "java"
 require "logstash/inputs/base"
 require "logstash/util/socket_peer"
 require "logstash-input-tcp_jars"
-require "logstash/inputs/tcp/decoder_impl"
-require "logstash/inputs/tcp/compat_ssl_options"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
 
 require "socket"
 require "openssl"
@@ -61,7 +60,13 @@ require "openssl"
 #     }
 class LogStash::Inputs::Tcp < LogStash::Inputs::Base
 
-  java_import org.logstash.tcp.InputLoop
+  java_import 'org.logstash.tcp.InputLoop'
+  java_import 'org.logstash.tcp.SslContextBuilder'
+
+  require_relative "tcp/decoder_impl"
+
+  # ecs_compatibility option, provided by Logstash core or the support adapter.
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
 
   config_name "tcp"
 
@@ -103,7 +108,8 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
   # Useful when the CA chain is not necessary in the system store.
   config :ssl_extra_chain_certs, :validate => :array, :default => []
 
-  # Validate client certificates against these authorities. You can define multiple files or paths. All the certificates will be read and added to the trust store.
+  # Validate client certificates against these authorities. You can define multiple files or paths.
+  # All the certificates will be read and added to the trust store.
   config :ssl_certificate_authorities, :validate => :array, :default => []
 
   # Instruct the socket to use TCP keep alives. Uses OS defaults for keep alive settings.
@@ -112,13 +118,6 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
   # Option to allow users to avoid DNS Reverse Lookup.
   config :dns_reverse_lookup_enabled, :validate => :boolean, :default => true
 
-  HOST_FIELD = "host".freeze
-  HOST_IP_FIELD = "[@metadata][ip_address]".freeze
-  PORT_FIELD = "port".freeze
-  PROXY_HOST_FIELD = "proxy_host".freeze
-  PROXY_PORT_FIELD = "proxy_port".freeze
-  SSLSUBJECT_FIELD = "sslsubject".freeze
-
   # Monkey patch TCPSocket and SSLSocket to include socket peer
   # @private
   def self.patch_socket_peer!
@@ -133,6 +132,8 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
   def initialize(*args)
     super(*args)
 
+    setup_fields!
+
     self.class.patch_socket_peer!
 
     # threadsafe socket bookkeeping
@@ -148,10 +149,7 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
     fix_streaming_codecs
 
     if server?
-      ssl_context = get_ssl_context(SslOptions)
-
-
-      @loop = InputLoop.new(@host, @port, DecoderImpl.new(@codec, self), @tcp_keep_alive, ssl_context)
+      @loop = InputLoop.new(@host, @port, DecoderImpl.new(@codec, self), @tcp_keep_alive, java_ssl_context)
     end
   end
 
@@ -188,8 +186,8 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
                     proxy_port, tbuf, socket)
     codec.decode(tbuf) do |event|
       if @proxy_protocol
-        event.set(PROXY_HOST_FIELD, proxy_address) unless event.get(PROXY_HOST_FIELD)
-        event.set(PROXY_PORT_FIELD, proxy_port) unless event.get(PROXY_PORT_FIELD)
+        event.set(@field_proxy_host, proxy_address) unless event.get(@field_proxy_host)
+        event.set(@field_proxy_port, proxy_port) unless event.get(@field_proxy_port)
       end
       enqueue_decorated(event, client_ip_address, client_address, client_port, socket)
     end
@@ -262,14 +260,24 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
   end
 
   def enqueue_decorated(event, client_ip_address, client_address, client_port, socket)
-    event.set(HOST_FIELD, client_address) unless event.get(HOST_FIELD)
-    event.set(HOST_IP_FIELD, client_ip_address) unless event.get(HOST_IP_FIELD)
-    event.set(PORT_FIELD, client_port) unless event.get(PORT_FIELD)
-    event.set(SSLSUBJECT_FIELD, socket.peer_cert.subject.to_s) if socket && @ssl_enable && @ssl_verify && event.get(SSLSUBJECT_FIELD).nil?
+    event.set(@field_host, client_address) unless event.get(@field_host)
+    event.set(@field_host_ip, client_ip_address) unless event.get(@field_host_ip)
+    event.set(@field_port, client_port) unless event.get(@field_port)
+    event.set(@field_sslsubject, socket.peer_cert.subject.to_s) if socket && @ssl_enable && @ssl_verify && event.get(@field_sslsubject).nil?
     decorate(event)
     @output_queue << event
   end
 
+  # setup the field names, with respect to ECS compatibility.
+  def setup_fields!
+    @field_host       = ecs_select[disabled: "host",                    v1: "[@metadata][input][tcp][source][name]"        ].freeze
+    @field_host_ip    = ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][tcp][source][ip]"          ].freeze
+    @field_port       = ecs_select[disabled: "port",                    v1: "[@metadata][input][tcp][source][port]"        ].freeze
+    @field_proxy_host = ecs_select[disabled: "proxy_host",              v1: "[@metadata][input][tcp][proxy][ip]"           ].freeze
+    @field_proxy_port = ecs_select[disabled: "proxy_port",              v1: "[@metadata][input][tcp][proxy][port]"         ].freeze
+    @field_sslsubject = ecs_select[disabled: "sslsubject",              v1: "[@metadata][input][tcp][tls][client][subject]"].freeze
+  end
+
   def server?
     @mode == "server"
   end
@@ -320,7 +328,7 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
 
     socket
   rescue OpenSSL::SSL::SSLError => e
-    @logger.error("SSL Error", :exception => e, :backtrace => e.backtrace)
+    @logger.error("SSL Error", :message => e.message, :exception => e.class, :backtrace => e.backtrace)
     # catch all rescue nil on close to discard any close errors or invalid socket
     socket.close rescue nil
     sleep(1) # prevent hammering peer
@@ -362,15 +370,33 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
     @socket_mutex.synchronize{@connection_sockets.keys.dup}
   end
 
-  def get_ssl_context(options_class)
-    ssl_context = options_class.builder
-      .set_is_ssl_enabled(@ssl_enable)
+  def java_ssl_context
+    SslContextBuilder.new
+      .set_ssl_enabled(@ssl_enable)
       .set_should_verify(@ssl_verify)
       .set_ssl_cert(@ssl_cert)
       .set_ssl_key(@ssl_key)
-      .set_ssl_key_passphrase(@ssl_key_passphrase.value)
+      .set_ssl_key_password(@ssl_key_passphrase.value)
       .set_ssl_extra_chain_certs(@ssl_extra_chain_certs.to_java(:string))
       .set_ssl_certificate_authorities(@ssl_certificate_authorities.to_java(:string))
-      .build.toSslContext()
+      .build_context
+  rescue java.lang.IllegalArgumentException => e
+    @logger.error("SSL configuration invalid", error_details(e))
+    raise LogStash::ConfigurationError, e
+  rescue java.lang.Exception => e
+    @logger.error("SSL configuration failed", error_details(e, true))
+    raise e
+  end
+
+  def error_details(e, trace = false)
+    error_details = { :exception => e.class, :message => e.message }
+    error_details[:backtrace] = e.backtrace if trace || @logger.debug?
+    cause = e.cause
+    if cause && e != cause
+      error_details[:cause] = { :exception => cause.class, :message => cause.message }
+      error_details[:cause][:backtrace] = cause.backtrace if trace || @logger.debug?
+    end
+    error_details
   end
+
 end

data/lib/logstash/inputs/tcp/decoder_impl.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 require 'java'
 
-class DecoderImpl
+class LogStash::Inputs::Tcp::DecoderImpl
 
   include org.logstash.tcp.Decoder
 
@@ -24,7 +24,7 @@ class DecoderImpl
   end
 
   def copy
-    DecoderImpl.new(@codec.clone, @tcp)
+    self.class.new(@codec.clone, @tcp)
   end
 
   def flush
@@ -41,16 +41,17 @@ class DecoderImpl
         @tcp.logger.error("Invalid proxy protocol header label", :header => pp_hdr)
         raise IOError.new("Invalid proxy protocol header label #{pp_hdr.inspect}")
       else
-        @proxy_address = pp_info[3]
-        @proxy_port = pp_info[5]
-        @address = pp_info[2]
-        @port = pp_info[4]
+        @proxy_address = pp_info[3] # layer 3 destination address (proxy's receiving address)
+        @proxy_port = pp_info[5] # TCP destination port (proxy's receiving port)
+        @ip_address = pp_info[2] # layer 3 source address (outgoing ip of sender)
+        @address = extract_host_name(@ip_address)
+        @port = pp_info[4] # TCP source port (outgoing port on sender [probably random])
       end
     else
       filtered = received
-      @ip_address = channel_addr.get_address.get_host_address
-      @address = extract_host_name(channel_addr)
-      @port = channel_addr.get_port
+      @ip_address = channel_addr.get_address.get_host_address # ip address of sender
+      @address = extract_host_name(channel_addr) # name _or_ address of sender
+      @port = channel_addr.get_port # outgoing port of sender (probably random)
     end
     @first_read = false
     filtered
@@ -58,6 +59,8 @@ class DecoderImpl
 
   private
   def extract_host_name(channel_addr)
+    channel_addr = java.net.InetSocketAddress.new(channel_addr, 0) if channel_addr.kind_of?(String)
+
     return channel_addr.get_host_string unless @tcp.dns_reverse_lookup_enabled?
 
     channel_addr.get_host_name

data/logstash-input-tcp.gemspec

@@ -21,6 +21,12 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2'
+
+  s.add_runtime_dependency 'logstash-core', '>= 6.7.0'
+
+  # we depend on bouncycastle's bcpkix-jdk15on being on the class-path
+  s.add_runtime_dependency 'jruby-openssl', '>= 0.10.2', '< 0.12'
 
   # line vs streaming codecs required for fix_streaming_codecs
   # TODO: fix_streaming_codecs should be refactored to not

data/spec/inputs/tcp_spec.rb

@@ -15,9 +15,11 @@ java_import "io.netty.handler.ssl.util.SelfSignedCertificate"
 
 require_relative "../spec_helper"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+
 #Cabin::Channel.get(LogStash).subscribe(STDOUT)
 #Cabin::Channel.get(LogStash).level = :debug
-describe LogStash::Inputs::Tcp do
+describe LogStash::Inputs::Tcp, :ecs_compatibility_support do
 
   def get_port
     begin
@@ -52,160 +54,176 @@ describe LogStash::Inputs::Tcp do
     end
   end
 
-  it "should read plain with unicode" do
-    event_count = 10
-    conf = <<-CONFIG
-      input {
-        tcp {
-          port => #{port}
+  ecs_compatibility_matrix(:disabled,:v1, :v8 => :v1) do |ecs_select|
+    before(:each) do
+      allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+    end
+
+    it "should read plain with unicode" do
+      event_count = 10
+      conf = <<-CONFIG
+        input {
+          tcp {
+            port => #{port}
+          }
         }
-      }
-    CONFIG
+      CONFIG
+
+      host = 'localhost'
+      events = input(conf) do |pipeline, queue|
+        socket = Stud::try(5.times) { TCPSocket.new(host, port) }
+        event_count.times do |i|
+          # unicode smiley for testing unicode support!
+          socket.puts("#{i} ☹")
+          socket.flush
+        end
+        socket.close
 
-    host = 'localhost'
-    events = input(conf) do |pipeline, queue|
-      socket = Stud::try(5.times) { TCPSocket.new(host, port) }
-      event_count.times do |i|
-        # unicode smiley for testing unicode support!
-        socket.puts("#{i} ☹")
-        socket.flush
+        event_count.times.collect {queue.pop}
       end
-      socket.close
 
-      event_count.times.collect {queue.pop}
-    end
+      expect(events.length).to eq(event_count)
+      events = events.sort_by {|e| e.get("message")} # the ordering of events in the queue is highly timing-dependent
+      event_count.times do |i|
+        event = events[i]
 
-    insist { events.length } == event_count
-    events = events.sort_by {|e| e.get("message")} # the ordering of events in the queue is highly timing-dependent
-    event_count.times do |i|
-      event = events[i]
-      insist { event.get("message") } == "#{i} ☹"
-      insist { ["localhost","ip6-localhost"].includes? event.get("host") }
-      insist { event.get("[@metadata][ip_address]") } == '127.0.0.1'
+        aggregate_failures("event #{i}") do
+          expect(event.get("message")).to eq("#{i} ☹")
+          expect(event.get(ecs_select[disabled: "host", v1: "[@metadata][input][tcp][source][name]"])).to eq("localhost").or eq("ip6-localhost")
+          expect(event.get(ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][tcp][source][ip]"])).to eq('127.0.0.1')
+        end
+      end
     end
-  end
 
-  it "should handle PROXY protocol v1 connections" do
-    event_count = 10
-    conf = <<-CONFIG
-      input {
-        tcp {
-          proxy_protocol => true
-          port => '#{port}'
+    it "should handle PROXY protocol v1 connections" do
+      event_count = 10
+      conf = <<-CONFIG
+        input {
+          tcp {
+            proxy_protocol => true
+            port => '#{port}'
+          }
         }
-      }
-    CONFIG
+      CONFIG
 
-    events = input(conf) do |pipeline, queue|
-      socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
-      socket.puts("PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678\r");
-      socket.flush
-      event_count.times do |i|
-        # unicode smiley for testing unicode support!
-        socket.puts("#{i} ☹")
+      events = input(conf) do |pipeline, queue|
+        socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
+        socket.puts("PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678\r");
         socket.flush
-      end
-      socket.close
+        event_count.times do |i|
+          # unicode smiley for testing unicode support!
+          socket.puts("#{i} ☹")
+          socket.flush
+        end
+        socket.close
 
-      event_count.times.collect {queue.pop}
-    end
+        event_count.times.collect {queue.pop}
+      end
 
-    insist { events.length } == event_count
-    events = events.sort_by {|e| e.get("message")} # the ordering of events in the queue is highly timing-dependent
-    event_count.times do |i|
-      insist { events[i].get("message") } == "#{i} ☹"
-      insist { events[i].get("host") } == "1.2.3.4"
-      insist { events[i].get("port") } == "1234"
-      insist { events[i].get("proxy_host") } == "5.6.7.8"
-      insist { events[i].get("proxy_port") } == "5678"
+      expect(events.length).to eq(event_count)
+      events = events.sort_by {|e| e.get("message")} # the ordering of events in the queue is highly timing-dependent
+      events.each_with_index do |event, i|
+        aggregate_failures("event #{i}") do
+          expect(event.get("message")).to eq("#{i} ☹")
+          expect(event.get(ecs_select[disabled: "host",                    v1: "[@metadata][input][tcp][source][name]"])).to eq('1.2.3.4')
+          expect(event.get(ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][tcp][source][ip]"  ])).to eq('1.2.3.4')
+          expect(event.get(ecs_select[disabled: "port",                    v1: "[@metadata][input][tcp][source][port]"])).to eq('1234')
+          expect(event.get(ecs_select[disabled: "proxy_host",              v1: "[@metadata][input][tcp][proxy][ip]"  ])).to eq('5.6.7.8')
+          expect(event.get(ecs_select[disabled: "proxy_port",              v1: "[@metadata][input][tcp][proxy][port]"  ])).to eq('5678')
+        end
+      end
     end
-  end
 
-  it "should read events with plain codec and ISO-8859-1 charset" do
-    charset = "ISO-8859-1"
-    conf = <<-CONFIG
-      input {
-        tcp {
-          port => #{port}
-          codec => plain { charset => "#{charset}" }
+    it "should read events with json codec" do
+      conf = <<-CONFIG
+        input {
+          tcp {
+            port => #{port}
+            codec => json
+          }
         }
+      CONFIG
+
+      data = {
+        "hello" => "world",
+        "foo" => [1,2,3],
+        "baz" => { "1" => "2" },
+        "host" => "example host"
       }
-    CONFIG
 
-    event = input(conf) do |pipeline, queue|
-      socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
-      text = "\xA3" # the £ symbol in ISO-8859-1 aka Latin-1
-      text.force_encoding("ISO-8859-1")
-      socket.puts(text)
-      socket.close
+      event = input(conf) do |pipeline, queue|
+        socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
+        socket.puts(LogStash::Json.dump(data))
+        socket.close
 
-      queue.pop
-    end
+        queue.pop
+      end
 
-    # Make sure the 0xA3 latin-1 code converts correctly to UTF-8.
-    insist { event.get("message").size } == 1
-    insist { event.get("message").bytesize } == 2
-    insist { event.get("message") } == "£"
-  end
+      insist { event.get("hello") } == data["hello"]
+      insist { event.get("foo").to_a } == data["foo"] # to_a to cast Java ArrayList produced by JrJackson
+      insist { event.get("baz") } == data["baz"]
 
-  it "should read events with json codec" do
-    conf = <<-CONFIG
-      input {
-        tcp {
-          port => #{port}
-          codec => json
+      # Make sure the tcp input, w/ json codec, uses the event's 'host' value,
+      # if present, instead of providing its own
+      insist { event.get("host") } == data["host"]
+    end
+
+    it "should read events with json codec (testing 'host' handling)" do
+      conf = <<-CONFIG
+        input {
+          tcp {
+            port => #{port}
+            codec => json
+          }
         }
+      CONFIG
+
+      data = {
+        "hello" => "world"
       }
-    CONFIG
 
-    data = {
-      "hello" => "world",
-      "foo" => [1,2,3],
-      "baz" => { "1" => "2" },
-      "host" => "example host"
-    }
+      event = input(conf) do |pipeline, queue|
+        socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
+        socket.puts(LogStash::Json.dump(data))
+        socket.close
 
-    event = input(conf) do |pipeline, queue|
-      socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
-      socket.puts(LogStash::Json.dump(data))
-      socket.close
+        queue.pop
+      end
 
-      queue.pop
+      aggregate_failures("event") do
+        expect(event.get("hello")).to eq(data["hello"])
+        expect(event).to include(ecs_select[disabled: "host",                    v1: "[@metadata][input][tcp][source][name]"])
+        expect(event).to include(ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][tcp][source][ip]"  ])
+      end
     end
-
-    insist { event.get("hello") } == data["hello"]
-    insist { event.get("foo").to_a } == data["foo"] # to_a to cast Java ArrayList produced by JrJackson
-    insist { event.get("baz") } == data["baz"]
-
-    # Make sure the tcp input, w/ json codec, uses the event's 'host' value,
-    # if present, instead of providing its own
-    insist { event.get("host") } == data["host"]
   end
 
-  it "should read events with json codec (testing 'host' handling)" do
+  it "should read events with plain codec and ISO-8859-1 charset" do
+    charset = "ISO-8859-1"
     conf = <<-CONFIG
-      input {
-        tcp {
-          port => #{port}
-          codec => json
+        input {
+          tcp {
+            port => #{port}
+            codec => plain { charset => "#{charset}" }
+          }
         }
-      }
     CONFIG
 
-    data = {
-      "hello" => "world"
-    }
-
     event = input(conf) do |pipeline, queue|
       socket = Stud::try(5.times) { TCPSocket.new("127.0.0.1", port) }
-      socket.puts(LogStash::Json.dump(data))
+      text = "\xA3" # the £ symbol in ISO-8859-1 aka Latin-1
+      text.force_encoding("ISO-8859-1")
+      socket.puts(text)
       socket.close
 
       queue.pop
     end
 
-    insist { event.get("hello") } == data["hello"]
-    insist { event }.include?("host")
+    # Make sure the 0xA3 latin-1 code converts correctly to UTF-8.
+    aggregate_failures("event") do
+      expect(event.get("message")).to have_attributes(size: 1, bytesize: 2, encoding: Encoding.find("UTF-8"))
+      expect(event.get("message")).to eq("£")
+    end
   end
 
   it "should read events with json_lines codec" do

data/version

@@ -1 +1 @@
-6.0.8
+6.2.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-tcp
 version: !ruby/object:Gem::Version
-  version: 6.0.8
+  version: 6.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-23 00:00:00.000000000 Z
+date: 2021-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,54 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.7.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.7.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  name: jruby-openssl
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -171,15 +219,13 @@ files:
 - NOTICE.TXT
 - README.md
 - docs/index.asciidoc
-- docs/testfile.asciidoc
 - lib/logstash-input-tcp_jars.rb
 - lib/logstash/inputs/tcp.rb
-- lib/logstash/inputs/tcp/compat_ssl_options.rb
 - lib/logstash/inputs/tcp/decoder_impl.rb
 - logstash-input-tcp.gemspec
 - spec/inputs/tcp_spec.rb
 - spec/spec_helper.rb
-- vendor/jar-dependencies/org/logstash/inputs/logstash-input-tcp/6.0.8/logstash-input-tcp-6.0.8.jar
+- vendor/jar-dependencies/org/logstash/inputs/logstash-input-tcp/6.2.0/logstash-input-tcp-6.2.0.jar
 - version
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:

data/docs/testfile.asciidoc

@@ -1,30 +0,0 @@
-:plugin: tcp-test
-:type: input
-:default_codec: line
-
-///////////////////////////////////////////
-START - GENERATED VARIABLES, DO NOT EDIT!
-///////////////////////////////////////////
-:version: %VERSION%
-:release_date: %RELEASE_DATE%
-:changelog_url: %CHANGELOG_URL%
-:include_path: ../../../../logstash/docs/include
-///////////////////////////////////////////
-END - GENERATED VARIABLES, DO NOT EDIT!
-///////////////////////////////////////////
-
-[id="plugins-{type}s-{plugin}"]
-
-=== Tcp-test input plugin
-
-include::{include_path}/plugin_header.asciidoc[]
-
-==== Description
-
-This is only a test. 
-
-
-[id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
-
-:default_codec!:

data/lib/logstash/inputs/tcp/compat_ssl_options.rb

@@ -1,147 +0,0 @@
-require 'openssl'
-require "logstash/util/loggable"
-
-# Simulate a normal SslOptions builder:
-#
-#     ssl_context = SslOptions.builder
-#       .set_is_ssl_enabled(@ssl_enable)
-#       .set_should_verify(@ssl_verify)
-#       .set_ssl_cert(@ssl_cert)
-#       .set_ssl_key(@ssl_key)
-#       .set_ssl_key_passphrase(@ssl_key_passphrase.value)
-#       .set_ssl_extra_chain_certs(@ssl_extra_chain_certs.to_java(:string))
-#       .set_ssl_certificate_authorities(@ssl_certificate_authorities.to_java(:string))
-#       .build.toSslContext()
-class SslOptions
-  include LogStash::Util::Loggable
-
-  java_import 'io.netty.handler.ssl.ClientAuth'
-  java_import 'io.netty.handler.ssl.SslContextBuilder'
-  java_import 'java.security.cert.X509Certificate'
-  java_import 'javax.crypto.Cipher'
-  java_import 'org.bouncycastle.asn1.pkcs.PrivateKeyInfo'
-  java_import 'org.bouncycastle.jce.provider.BouncyCastleProvider'
-  java_import 'org.bouncycastle.openssl.PEMKeyPair'
-  java_import 'org.bouncycastle.openssl.PEMParser'
-  java_import 'org.bouncycastle.openssl.PEMEncryptedKeyPair'
-  java_import 'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter'
-  java_import 'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder'
-  java_import 'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder'
-  java_import 'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo'
-
-  def self.builder
-    new
-  end
-
-  def set_is_ssl_enabled(boolean)
-    @ssl_enabled = boolean
-    self
-  end
-
-  def set_should_verify(boolean)
-    @ssl_verify = boolean
-    self
-  end
-
-  def set_ssl_cert(path)
-    @ssl_cert_path = path
-    self
-  end
-
-  def set_ssl_key(path)
-    @ssl_key_path = path
-    self
-  end
-
-  def set_ssl_key_passphrase(passphrase)
-    @ssl_key_passphrase = passphrase
-    self
-  end
-
-  def set_ssl_extra_chain_certs(certs)
-    @ssl_extra_chain_certs = certs
-    self
-  end
-
-  def set_ssl_certificate_authorities(certs)
-    @ssl_certificate_authorities = certs
-    self
-  end
-
-  def build; self; end
-
-  def toSslContext
-    return nil unless @ssl_enabled
-
-    # Check key strength
-    logger.warn("JCE Unlimited Strength Jurisdiction Policy not installed - max key length is 128 bits") unless Cipher.getMaxAllowedKeyLength("AES") > 128
-    # create certificate object
-    cf = java.security.cert.CertificateFactory.getInstance("X.509")
-    cert_chain = []
-    fetch_certificates_from_file(@ssl_cert_path, cf) do |cert|
-      cert_chain << cert
-    end
-
-    # convert key from pkcs1 to pkcs8 and get PrivateKey object
-    pem_parser = PEMParser.new(java.io.FileReader.new(@ssl_key_path))
-    java.security.Security.addProvider(BouncyCastleProvider.new)
-    converter = JcaPEMKeyConverter.new
-    case obj = pem_parser.readObject
-    when PEMKeyPair # unencrypted pkcs#1
-      private_key = converter.getKeyPair(obj).private
-    when PrivateKeyInfo # unencrypted pkcs#8
-      private_key = converter.getPrivateKey(obj)
-    when PEMEncryptedKeyPair # encrypted pkcs#1
-      key_char_array = @ssl_key_passphrase.to_java.toCharArray
-      decryptor = JcePEMDecryptorProviderBuilder.new.build(key_char_array)
-      key_pair = obj.decryptKeyPair(decryptor)
-      private_key = converter.getKeyPair(key_pair).private
-    when PKCS8EncryptedPrivateKeyInfo # encrypted pkcs#8
-      key_char_array = @ssl_key_passphrase.to_java.toCharArray
-      key = JceOpenSSLPKCS8DecryptorProviderBuilder.new.build(key_char_array)
-      private_key = converter.getPrivateKey(obj.decryptPrivateKeyInfo(key))
-    else
-      raise "Could not recognize 'ssl_key' format. Class: #{obj.class}"
-    end
-
-    @ssl_extra_chain_certs.each do |file|
-      fetch_certificates_from_file(file, cf) do |cert|
-        cert_chain << cert
-      end
-    end
-    sslContextBuilder = SslContextBuilder.forServer(private_key, @ssl_key_passphrase, cert_chain.to_java(X509Certificate))
-
-    trust_certs = []
-
-    @ssl_certificate_authorities.each do |file|
-      fetch_certificates_from_file(file, cf) do |cert|
-        trust_certs << cert
-      end
-    end
-
-    if trust_certs.any?
-      sslContextBuilder.trustManager(trust_certs.to_java(X509Certificate))
-    end
-
-    sslContextBuilder.clientAuth(@ssl_verify ? ClientAuth::REQUIRE : ClientAuth::NONE)
-    sslContextBuilder.build()
-  end
-
-  private
-  def fetch_certificates_from_file(file, cf)
-    fis = java.io.FileInputStream.new(file)
-
-    while (fis.available > 0) do
-      cert = generate_certificate(cf, fis)
-      yield cert if cert
-    end
-  ensure
-    fis.close if fis
-  end
-
-  def generate_certificate(cf, fis)
-    cf.generateCertificate(fis)
-  rescue Java::JavaSecurityCert::CertificateException => e
-    raise e unless e.cause.message == "Empty input"
-  end
-end