logstash-input-beats 6.4.4-java → 6.6.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc12eb3ae5765eeaa163d535c54da36260a0ce9cb2fb7b2561cee15ae3095208
-  data.tar.gz: 01010c8ea94c4e536c817997206a2b2c7422d9e9a8c66dbfe731467d26d888be
+  metadata.gz: 92357d87addd898c82f05a241738a7b83debb31e2ae55197f5c2dd9003a75781
+  data.tar.gz: fbfd41f7cc59c46d1ca43a8b3a761b9c1dff1026d23aae5bcf5f9e2eb1b3c94f
 SHA512:
-  metadata.gz: 1942d31a3e5999c9852789a4b9fb8ea97fc3ab4edde985bce7db484e3f3e6a7ce93313beb08a7819450285fe54fc2ac87e637e1d3574114c325100402432378d
-  data.tar.gz: dd3d56c917de244d33c986e46dece26bc449cbfd5a4eaf408ef03140ecb7b85768ae4e02ee70fe6d5dda6f705ace390ca24ea219286ef348e4dac23de6b8e4c0
+  metadata.gz: 33330e2fe60093cde6c177483e74e2137064a232ad8beff1faa682433f342cb079817c831d6f22c8356509f9c51f6d53ee63cc7bf554e1123a1b3d48e375ea9a
+  data.tar.gz: 48d537c69e2bdcef820336dd55f8ec292a643503beaac9b594d4d41e1cb411567fc6274b7d17f6eca607bd83288c459beeb0d27265bd7d6be4997fbad8768c07

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 6.6.0
+ - Reviewed and deprecated SSL settings to comply with Logstash's naming convention [#470](https://github.com/logstash-plugins/logstash-input-beats/pull/470)
+   - Deprecated `ssl` in favor of `ssl_enabled`
+   - Deprecated `ssl_verify_mode` in favor of `ssl_client_authentication`
+
+## 6.5.0
+ - An enrichment `enrich` option added to control ECS passthrough. `ssl_peer_metadata` and `include_codec_tag` configurations are deprecated and can be managed through the `enrich`  [#464](https://github.com/logstash-plugins/logstash-input-beats/pull/464)
+
 ## 6.4.4
  - Updates Netty dependency to 4.1.87 [#466](https://github.com/logstash-plugins/logstash-input-beats/pull/466)
 

data/VERSION

@@ -1 +1 @@
-6.4.4
+6.6.0

data/docs/index.asciidoc

@@ -143,26 +143,65 @@ endif::[]
 
 
 [id="plugins-{type}s-{plugin}-ecs_metadata"]
-==== Event Metadata and the Elastic Common Schema (ECS)
+==== Event enrichment and the Elastic Common Schema (ECS)
 
-When decoding {plugin-uc} events, this plugin adds two fields related to the event:
-the deprecated `host` which contains the `hostname` provided by {plugin-uc} and the
-`ip_address` containing the remote address of the client's connection. When
-<<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled
-these are now moved in ECS compatible namespace. Here's how
-<<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> affects
-output.  
+When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing. 
+You can use the <<plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories.
 
-[cols="<l,<l,e,<e"]
+The location of these enrichment fields depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled:
+
+- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure.
+- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema.
+
+
+.`source_metadata`
+[cols="<l,<l,<e",caption="Enrichment category:"]
+|=======================================================================
+|ECS `v1`, `v8` |ECS `disabled` |Description
+
+|[@metadata][input][beats][host][name]
+|[host]
+|Name or address of the {plugin-singular} host
+
+|[@metadata][input][beats][host][ip]
+|[@metadata][ip_address]
+|IP address of the {plugin-uc} client that connected to this input
+|=======================================================================
+
+.`ssl_peer_metadata`
+[cols="<l,<l,<e",caption="Enrichment category:"]
+|=======================================================================
+|ECS `v1`, `v8` |ECS `disabled` |Description
+
+|[@metadata][tls_peer][status]
+|[@metadata][tls_peer][status]
+|Contains "verified" or "unverified" label; available when SSL is enabled.
+
+|[@metadata][input][beats][tls][version_protocol]
+|[@metadata][tls_peer][protocol]
+|Contains the TLS version used (such as `TLSv1.2`); available when SSL status is "verified"
+
+|[@metadata][input][beats][tls][client][subject]
+|[@metadata][tls_peer][subject]
+|Contains the identity name of the remote end (such as `CN=artifacts-no-kpi.elastic.co`); available when SSL status is "verified"
+
+|[@metadata][input][beats][tls][cipher]
+|[@metadata][tls_peer][cipher_suite]
+|Contains the name of cipher suite used (such as `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`); available when SSL status is "verified"
+|=======================================================================
+
+.`codec_metadata`
+[cols="<l,<l,<e",caption="Enrichment category:"]
 |=======================================================================
-|ECS `disabled` |ECS `v1`, `v8` |Availability |Description
-
-|[host]                  |[@metadata][input][beats][host][name]   |Always       |Name or address of the {plugin-singular} host
-|[@metadata][ip_address] |[@metadata][input][beats][host][ip]     |Always       |IP address of the {plugin-uc} client
-|[@metadata][tls_peer][status] | [@metadata][tls_peer][status] | When SSL related fields are populated | Contains "verified"/"unverified" labels in `disabled`, `true`/`false` in `v1`/`v8`
-|[@metadata][tls_peer][protocol] | [@metadata][input][beats][tls][version_protocol] | When SSL status is "verified" | Contains the TLS version used (e.g. `TLSv1.2`)
-|[@metadata][tls_peer][subject] | [@metadata][input][beats][tls][client][subject] | When SSL status is "verified" | Contains the identity name of the remote end (e.g. `CN=artifacts-no-kpi.elastic.co`)
-|[@metadata][tls_peer][cipher_suite] | [@metadata][input][beats][tls][cipher] | When SSL status is "verified" | Contains the name of cipher suite used (e.g. `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`)
+|ECS `v1`, `v8` |ECS `disabled` |Description
+
+|[tag]
+|[tag]
+|Contains `beats_input_codec_XXX_applied` where `XXX` is the name of the codec
+
+|[event][original]
+e|N/A
+|When ECS is enabled, even if `[event][original]` field does not already exist on the event being processed, this plugin's *default codec* ensures that the field is populated using the bytes as-processed.
 |=======================================================================
 
 [id="plugins-{type}s-{plugin}-options"]
@@ -177,19 +216,23 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
+| <<plugins-{type}s-{plugin}-enrich>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-executor_threads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-ssl_client_authentication>> |<<string,string>>, one of `["none", "optional", "required"]`|No
+| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ssl_handshake_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-ssl_key_passphrase>> |<<password,password>>|No
-| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<array,array>>|No
-| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|No
+| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|__Deprecated__
 | <<plugins-{type}s-{plugin}-tls_max_version>> |<<number,number>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-tls_min_version>> |<<number,number>>|__Deprecated__
 |=======================================================================
@@ -239,14 +282,72 @@ Close Idle clients after X seconds of inactivity.
 
 Refer to <<plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information.
 
+[id="plugins-{type}s-{plugin}-enrich"]
+===== `enrich`
+
+* Value type is <<string,string>>
+** A <<list,list>> can also be provided
+** Configures which enrichments are applied to each event
+** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments.
+** Supported values are:
++
+[cols="2l,5"]
+|=======================================================================
+|Enrichment | Description
+
+| codec_metadata    | Information about how the codec transformed a sequence of bytes into
+                      this Event, such as _which_ codec was used. Also, if no <<codec>> is
+                      explicitly specified, _excluding_ `codec_metadata` from `enrich` will
+                      disable `ecs_compatibility` for this plugin.
+| source_metadata   | Information about the _source_ of the event, such as the IP address
+                      of the inbound connection this input received the event from and the
+                      name of the Logstash host that processed the event
+| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from,
+                      such as identity information from the SSL client certificate that was
+                      presented when establishing a connection to this input
+| all               | _alias_ to include _all_ available enrichments (including additional
+                      enrichments introduced in future versions of this plugin)
+| none              | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_
+                      defining <<codec>> with this option will not disable the `ecs_compatibility`,
+                      instead it relies on pipeline or codec `ecs_compatibility` configuration.
+|=======================================================================
+
+
+**Example:**
+
+This configuration disables _all_ enrichments:
+
+["source",subs="attributes"]
+--------------------------------------------------
+input {
+  beats {
+    port => 5044
+    enrich => none
+  }
+}
+--------------------------------------------------
+
+Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others):
+
+
+["source",subs="attributes"]
+--------------------------------------------------
+input {
+  beats {
+    port => 5044
+    enrich => [source_metadata, ssl_peer_metadata]
+  }
+}
+--------------------------------------------------
+
 [id="plugins-{type}s-{plugin}-executor_threads"]
 ===== `executor_threads`
 
   * Value type is <<number,number>>
   * Default value is equal to the number of CPU cores (1 executor thread per CPU core).
 
-The number of threads to be used to process incoming beats requests.
-By default, the Beats Input creates a number of threads equal to the number of CPU cores.
+The number of threads to be used to process incoming {plugin-uc} requests.
+By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores.
 These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management. 
 Parsing the Lumberjack protocol is offloaded to a dedicated thread pool.
 
@@ -268,6 +369,8 @@ The IP address to listen on.
 [id="plugins-{type}s-{plugin}-include_codec_tag"]
 ===== `include_codec_tag`
 
+deprecated[6.5.0, Replaced by <<plugins-{type}s-{plugin}-enrich>>]
+
   * Value type is <<boolean,boolean>>
   * Default value is `true`
 
@@ -284,6 +387,7 @@ The port to listen on.
 
 [id="plugins-{type}s-{plugin}-ssl"]
 ===== `ssl`
+deprecated[6.6.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -308,8 +412,8 @@ SSL certificate to use.
 
 Validate client certificates against these authorities.
 You can define multiple files or paths. All the certificates will
-be read and added to the trust store. You need to configure the `ssl_verify_mode`
-to `peer` or `force_peer` to enable the verification.
+be read and added to the trust store. You need to configure the <<plugins-{type}s-{plugin}-ssl_client_authentication>>
+to `optional` or `required` to enable the verification.
 
 [id="plugins-{type}s-{plugin}-ssl_cipher_suites"]
 ===== `ssl_cipher_suites`
@@ -322,6 +426,27 @@ This default list applies for OpenJDK 11.0.14 and higher.
 For older JDK versions, the default list includes only suites supported by that version.
 For example, the ChaCha20 family of ciphers is not supported in older versions.
 
+[id="plugins-{type}s-{plugin}-ssl_client_authentication"]
+===== `ssl_client_authentication`
+
+* Value can be any of: `none`, `optional`, `required`
+* Default value is `"none"`
+
+Controls the server's behavior in regard to requesting a certificate from client connections:
+`required` forces a client to present a certificate, while `optional` requests a client certificate
+but the client is not required to present one. Defaults to `none`, which disables the client authentication.
+
+NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set.
+
+[id="plugins-{type}s-{plugin}-ssl_enabled"]
+===== `ssl_enabled`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring
+the <<plugins-{type}s-{plugin}-ssl_certificate>> and <<plugins-{type}s-{plugin}-ssl_key>> options.
+
 [id="plugins-{type}s-{plugin}-ssl_handshake_timeout"]
 ===== `ssl_handshake_timeout`
 
@@ -357,12 +482,14 @@ SSL key passphrase to use.
 [id="plugins-{type}s-{plugin}-ssl_peer_metadata"]
 ===== `ssl_peer_metadata`
 
+deprecated[6.5.0, Replaced by <<plugins-{type}s-{plugin}-enrich>>]
+
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
 Enables storing client certificate information in event's metadata.
 
-This option is only valid when `ssl_verify_mode` is set to `peer` or `force_peer`.
+This option is only valid when <<plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`.
 
 [id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
 ===== `ssl_supported_protocols`
@@ -383,11 +510,13 @@ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.
 
 [id="plugins-{type}s-{plugin}-ssl_verify_mode"]
 ===== `ssl_verify_mode`
+deprecated[6.6.0, Replaced by <<plugins-{type}s-{plugin}-ssl_client_authentication>>]
 
   * Value can be any of: `none`, `peer`, `force_peer`
   * Default value is `"none"`
 
-By default the server doesn't do any client verification.
+By default, the server doesn't do any client verification. If the <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`.
 
 `peer` will make the server ask the client to provide a certificate.
 If the client provides a certificate, it will be validated.
@@ -395,7 +524,7 @@ If the client provides a certificate, it will be validated.
 `force_peer` will make the server ask the client to provide a certificate.
 If the client doesn't provide a certificate, the connection will be closed.
 
-This option needs to be used with `ssl_certificate_authorities` and a defined list of CAs.
+This option needs to be used with <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs.
 
 [id="plugins-{type}s-{plugin}-tls_max_version"]
 ===== `tls_max_version`
@@ -416,7 +545,6 @@ The minimum TLS version allowed for the encrypted connections.
 The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3
 
 
-
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 

data/lib/logstash/inputs/beats/message_listener.rb

@@ -32,11 +32,14 @@ module LogStash module Inputs class Beats
 
     def onNewMessage(ctx, message)
       hash = message.getData
-      ip_address = ip_address(ctx)
 
-      unless ip_address.nil? || hash['@metadata'].nil?
-        set_nested(hash, @input.field_hostip, ip_address)
+      if @input.include_source_metadata?
+        ip_address = ip_address(ctx)
+        unless ip_address.nil? || hash['@metadata'].nil?
+          set_nested(hash, @input.field_hostip, ip_address)
+        end
       end
+
       target_field = extract_target_field(hash)
 
       extract_tls_peer(hash, ctx)