logstash-input-beats 6.1.3-java → 6.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3d70597b1b447c581f1a7c6d9757aeff377cb57f9e0bf30ae62a1a82ae2e3d7
-  data.tar.gz: f2fbd686d950b35fb9d337330e226029b4dce5eaaf56e3ac31722cc6dbd849ac
+  metadata.gz: 16a55f0a53247ed30b7110d9f60dce425288fdd68c1bf936283698028bdfd421
+  data.tar.gz: d8cda82e3d3a8fb3ceef48a96f52674eb04496e7a0957043b34d5c5e741484fa
 SHA512:
-  metadata.gz: 892b7166ecd256432677e4bda1dbc2ebd4c9555fc2aecf26e73ed0d2747b2e736b6e1421582b3ec786ba6bef4c6a705c4c0a6d0b54423980c0f9f7ddbf4ddaf9
-  data.tar.gz: e8101ebad94b1924d68a2000b8c2b9c9996ef2634fcb1e8d87d445e64941b5278b6af8c24d7f93e3c271638028f36f258c717d9401595c213a3f99ec46f138ed
+  metadata.gz: e2869c6c573223e979f411c430ce1ddb6f65f6c967930b469c617a03b6a4fb0417a7eecfc23e2c8061599a72a07f656490bb76c9ceb5576f00613be0cdfde688
+  data.tar.gz: 38dd3f7bfb548565ecc047bc0e044e89796f2fcbdf8187f2b3a37fc1bb822a00d73c759713996dd69471fbaa1e334fd3a4b1e730adcb0093ccf6956f14f4549e

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 6.2.0
+ - ECS compatibility enablement: Adds alias to support upcoming ECS v8 with the existing ECS v1 implementation
+
+## 6.1.7
+ - [DOC] Remove limitations topic and link [#428](https://github.com/logstash-plugins/logstash-input-http/pull/428)
+
+## 6.1.6
+ - [DOC] Applied more attributes to manage plugin name in doc content, and implemented conditional text processing. [#423](https://github.com/logstash-plugins/logstash-input-http/pull/423)
+
+## 6.1.5
+ - Changed jar dependencies to reflect newer versions [#425](https://github.com/logstash-plugins/logstash-input-beats/pull/425)
+
+## 6.1.4
+ - Fix: reduce error logging on connection resets [#424](https://github.com/logstash-plugins/logstash-input-beats/pull/424)
+
 ## 6.1.3
  - Fix: safe-guard byte buf allocation [#420](https://github.com/logstash-plugins/logstash-input-beats/pull/420) 
  - Updated Jackson dependencies

data/VERSION

@@ -1 +1 @@
-6.1.3
+6.2.0

data/docs/index.asciidoc

@@ -2,6 +2,7 @@
 :type: input
 :default_codec: plain
 :plugin-uc: Beats
+:plugin-singular: Beat
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -18,21 +19,27 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === {plugin-uc} input plugin
 
+NOTE: The `input-elastic_agent` plugin is the next generation of the
+`input-beats` plugin. 
+They currently share code and a https://github.com/logstash-plugins/logstash-input-beats[common codebase].
+
 include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
 This input plugin enables Logstash to receive events from the
-https://www.elastic.co/products/beats[Elastic Beats] framework.
+{plugin-uc} framework.
 
 The following example shows how to configure Logstash to listen on port
 5044 for incoming {plugin-uc} connections and to index into Elasticsearch.
 
+//Example for Beats
+ifeval::["{plugin}"=="beats"]
 ["source","sh",subs="attributes"]
 -----
 
 input {
-  beats {
+  {plugin} {
     port => 5044
   }
 }
@@ -45,9 +52,8 @@ output {
 }
 -----
 <1> `%{[@metadata][beat]}` sets the first part of the index name to the value
-of the `beat` metadata field and `%{[@metadata][version]}` sets the second part to
-the {plugin-uc}'s version. For example:
-metricbeat-7.4.0.
+of the metadata field and `%{[@metadata][version]}` sets the second part to
+the {plugin-singular} version. For example: metricbeat-6.1.6.
 
 Events indexed into Elasticsearch with the Logstash configuration shown here
 will be similar to events directly indexed by {plugin-uc} into Elasticsearch.
@@ -56,14 +62,47 @@ NOTE: If ILM is not being used, set `index` to
 `%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so
 Logstash creates an index per day, based on the `@timestamp` value of the events
 coming from {plugin-uc}.
+endif::[]
+
+//Example for Elastic Agent
+ifeval::["{plugin}"!="beats"]
+["source","sh",subs="attributes"]
+-----
+
+input {
+  {plugin} {
+    port => 5044
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["http://localhost:9200"]
+    data_stream => "true"
+  }
+}
+-----
+
+Events indexed into Elasticsearch with the Logstash configuration shown here
+will be similar to events directly indexed by {plugin-uc} into Elasticsearch.
+endif::[]
+
+
+//Content for Beats
+ifeval::["{plugin}"=="beats"]
+[id="plugins-{type}s-{plugin}-multiline"]
+===== Multi-line events
 
-IMPORTANT: If you are shipping events that span multiple lines, you need to use
+If you are shipping events that span multiple lines, you need to use
 the {filebeat-ref}/multiline-examples.html[configuration options available in
 Filebeat] to handle multiline events before sending the event data to Logstash.
 You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec
 plugin] to handle multiline events. Doing so will result in the failure to start
 Logstash.
+endif::[]
 
+//Content for Beats
+ifeval::["{plugin}"=="beats"]
 [id="plugins-{type}s-{plugin}-versioned-indexes"]
 ==== Versioned indices
 
@@ -86,6 +125,7 @@ Logstash `@timestamp` field.
 
 This configuration results in daily index names like
 +filebeat-{logstash_version}-{localdate}+.
+endif::[]
 
 
 [id="plugins-{type}s-{plugin}-ecs_metadata"]
@@ -101,18 +141,18 @@ output.
 
 [cols="<l,<l,e,<e"]
 |=======================================================================
-|ECS disabled |ECS v1 |Availability |Description
+|ECS `disabled` |ECS `v1`, `v8` |Availability |Description
 
-|[host]                  |[@metadata][input][beats][host][name]   |Always       |Name or address of the beat host
-|[@metadata][ip_address] |[@metadata][input][beats][host][ip]     |Always       |IP address of the Beats client
-|[@metadata][tls_peer][status] | [@metadata][tls_peer][status] | When SSL related fields are populated | Contains "verified"/"unverified" labels in `disabled`, `true`/`false` in `v1`
+|[host]                  |[@metadata][input][beats][host][name]   |Always       |Name or address of the {plugin-singular} host
+|[@metadata][ip_address] |[@metadata][input][beats][host][ip]     |Always       |IP address of the {plugin-uc} client
+|[@metadata][tls_peer][status] | [@metadata][tls_peer][status] | When SSL related fields are populated | Contains "verified"/"unverified" labels in `disabled`, `true`/`false` in `v1`/`v8`
 |[@metadata][tls_peer][protocol] | [@metadata][input][beats][tls][version_protocol] | When SSL status is "verified" | Contains the TLS version used (e.g. `TLSv1.2`)
 |[@metadata][tls_peer][subject] | [@metadata][input][beats][tls][client][subject] | When SSL status is "verified" | Contains the identity name of the remote end (e.g. `CN=artifacts-no-kpi.elastic.co`)
 |[@metadata][tls_peer][cipher_suite] | [@metadata][input][beats][tls][cipher] | When SSL status is "verified" | Contains the name of cipher suite used (e.g. `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`)
 |=======================================================================
 
 [id="plugins-{type}s-{plugin}-options"]
-==== {plugin-uc} Input Configuration Options
+==== {plugin-uc} input configuration options
 
 This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
 
@@ -151,7 +191,7 @@ input plugins.
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-Flag to determine whether to add `host` field to event using the value supplied by the beat in the `hostname` field.
+Flag to determine whether to add `host` field to event using the value supplied by the {plugin-singular} in the `hostname` field.
 
 
 [id="plugins-{type}s-{plugin}-cipher_suites"]
@@ -176,7 +216,8 @@ Close Idle clients after X seconds of inactivity.
 * Value type is <<string,string>>
 * Supported values are:
 ** `disabled`: unstructured connection metadata added at root level
-** `v1`: structured connection metadata added under ECS compliant namespaces
+** `v1`: structured connection metadata added under ECS v1 compliant namespaces
+** `v8`: structured connection metadata added under ECS v8 compliant namespaces
 * Default value depends on which version of Logstash is running:
 ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
 ** Otherwise, the default value is `disabled`.

data/lib/logstash-input-beats_jars.rb

@@ -1,11 +1,11 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('io.netty', 'netty-all', '4.1.49.Final')
+require_jar('io.netty', 'netty-all', '4.1.65.Final')
 require_jar('org.javassist', 'javassist', '3.24.0-GA')
 require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.9.10')
 require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.10')
 require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.9.10.8')
 require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.10')
 require_jar('org.apache.logging.log4j', 'log4j-api', '2.11.1')
-require_jar('org.logstash.beats', 'logstash-input-beats', '6.1.3')
+require_jar('org.logstash.beats', 'logstash-input-beats', '6.2.0')

data/lib/logstash/inputs/beats.rb

@@ -6,6 +6,7 @@ require "logstash/codecs/multiline"
 require "logstash/util"
 require "logstash-input-beats_jars"
 require "logstash/plugin_mixins/ecs_compatibility_support"
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
 require_relative "beats/patch"
 
 # This input plugin enables Logstash to receive events from the
@@ -51,7 +52,9 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   require "logstash/inputs/beats/tls"
 
   # adds ecs_compatibility config which could be :disabled or :v1
-  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled,:v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled,:v1, :v8 => :v1)
+
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
 
   config_name "beats"
 

data/lib/logstash/inputs/beats/message_listener.rb

@@ -15,6 +15,8 @@ module LogStash module Inputs class Beats
 
     attr_reader :logger, :input, :connections_list
 
+    attr_reader :event_factory
+
     def initialize(queue, input)
       @connections_list = ThreadSafe::Hash.new
       @queue = queue
@@ -25,6 +27,7 @@ module LogStash module Inputs class Beats
 
       @nocodec_transformer = RawEventTransform.new(@input)
       @codec_transformer = DecodedEventTransform.new(@input)
+      @event_factory = input.event_factory
     end
 
     def onNewMessage(ctx, message)
@@ -39,7 +42,7 @@ module LogStash module Inputs class Beats
       extract_tls_peer(hash, ctx)
 
       if target_field.nil?
-        event = LogStash::Event.new(hash)
+        event = event_factory.new_event(hash)
         @nocodec_transformer.transform(event)
         @queue << event
       else

data/logstash-input-beats.gemspec

@@ -27,7 +27,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "thread_safe", "~> 0.3.5"
   s.add_runtime_dependency "logstash-codec-multiline", ">= 2.0.5"
   s.add_runtime_dependency 'jar-dependencies', '~> 0.3', '>= 0.3.4'
-  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
+  s.add_runtime_dependency 'logstash-mixin-event_support', '~>1.0'
 
   s.add_development_dependency "flores", "~>0.0.6"
   s.add_development_dependency "rspec"

data/spec/inputs/beats/decoded_event_transform_spec.rb

@@ -31,6 +31,7 @@ describe LogStash::Inputs::Beats::DecodedEventTransform do
 
   include_examples "Common Event Transformation", :disabled, "host"
   include_examples "Common Event Transformation", :v1, "[@metadata][input][beats][host][name]"
+  include_examples "Common Event Transformation", :v8, "[@metadata][input][beats][host][name]"
 
   it "tags the event" do
     expect(subject.get("tags")).to include("beats_input_codec_plain_applied")

data/spec/inputs/beats/event_transform_common_spec.rb

@@ -9,4 +9,5 @@ describe LogStash::Inputs::Beats::EventTransformCommon do
 
   include_examples "Common Event Transformation", :disabled, "host"
   include_examples "Common Event Transformation", :v1, "[@metadata][input][beats][host][name]"
+  include_examples "Common Event Transformation", :v8, "[@metadata][input][beats][host][name]"
 end

data/spec/inputs/beats/message_listener_spec.rb

@@ -211,6 +211,7 @@ describe LogStash::Inputs::Beats::MessageListener do
 
     it_behaves_like "when the message is from any libbeat", :disabled, "[@metadata][ip_address]"
     it_behaves_like "when the message is from any libbeat", :v1, "[@metadata][input][beats][host][ip]"
+    it_behaves_like "when the message is from any libbeat", :v8, "[@metadata][input][beats][host][ip]"
   end
 
   context "onException" do

data/spec/inputs/beats/raw_event_transform_spec.rb

@@ -20,6 +20,7 @@ describe LogStash::Inputs::Beats::RawEventTransform do
 
   include_examples "Common Event Transformation", :disabled, "host"
   include_examples "Common Event Transformation", :v1, "[@metadata][input][beats][host][name]"
+  include_examples "Common Event Transformation", :v8, "[@metadata][input][beats][host][name]"
 
   it "tags the event" do
     expect(subject.get("tags")).to include("beats_input_raw_event")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 6.1.3
+  version: 6.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-21 00:00:00.000000000 Z
+date: 2021-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -111,7 +111,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.3'
   name: logstash-mixin-ecs_compatibility_support
   prerelease: false
   type: :runtime
@@ -119,7 +119,21 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-event_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -283,9 +297,11 @@ files:
 - vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.9.10.8/jackson-databind-2.9.10.8.jar
 - vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.9.10/jackson-module-afterburner-2.9.10.jar
 - vendor/jar-dependencies/io/netty/netty-all/4.1.49.Final/netty-all-4.1.49.Final.jar
+- vendor/jar-dependencies/io/netty/netty-all/4.1.65.Final/netty-all-4.1.65.Final.jar
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.11.1/log4j-api-2.11.1.jar
 - vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar
 - vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.1.3/logstash-input-beats-6.1.3.jar
+- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.2.0/logstash-input-beats-6.2.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -308,8 +324,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Receives events from the Elastic Beats framework