logstash-input-beats 6.7.1-java → 6.8.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 633344379c679c34cba9b726bf6743f8dad7da7b65b4266e647d1539c3ef3a08
-  data.tar.gz: 340d1c62e61b64f6787a8e87df877cfd6073aa42ff9601504f21272119fc9937
+  metadata.gz: e795cf7e189345d1cf4ef96dfadc1185d2e73b63be07674654ce06342adaff86
+  data.tar.gz: 55bce211f32a39ebea8197b46ebdf83e0832d043a785d54dc9f6917bd5132ab2
 SHA512:
-  metadata.gz: 779e37e64ba62a4fd988a535866f393bb4797980eee02260b56ddf145194f398f1c2abfb5708d1939279bb4fcf2a6cba0b59b061ea59ea8c57d2a700d64e2e81
-  data.tar.gz: 39f78d631e00518b17af3d88109b3c41f58a3e9ef61d26e7e2a7c97eb0fb0c915b6f7d994b6975d5fd4d0f95cf5e24a54b7d959b373b501dacda8d639c19b04f
+  metadata.gz: 654adc75a917802031a32ba3d02279a135f0817e90833fe2bca54261fda59debe82072e0ecdf0f14bc5996afd644d0da36f9e9ed702456a76358d7d1d6a0e458
+  data.tar.gz: 72093407b408492a8a7bbaf57d5694a64f1b56c99e418726c91727d2fcc0b4ea7ca85da22344db0a51b4b6196b5d3ad3d78c2da213c8e653087ef4949c8e1e73

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 6.8.0
+  - Introduce expert only `event_loop_threads` to tune netty event loop threads count [#490](https://github.com/logstash-plugins/logstash-input-beats/pull/490)
+
+## 6.7.2
+  - Restore accidentally removed Lumberjack event parse source lines [#486](https://github.com/logstash-plugins/logstash-input-beats/pull/486)
+
 ## 6.7.1
   - bump netty to 4.1.100 and jackson to 2.15.3 [#484](https://github.com/logstash-plugins/logstash-input-beats/pull/484)
 

data/VERSION

@@ -1 +1 @@
-6.7.1
+6.8.0

data/docs/index.asciidoc

@@ -217,6 +217,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-enrich>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-event_loop_threads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-executor_threads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|__Deprecated__
@@ -339,6 +340,20 @@ input {
 }
 --------------------------------------------------
 
+[id="plugins-{type}s-{plugin}-event_loop_threads"]
+===== `event_loop_threads`
+
+* Value type is <<number,number>>
+* Defaults to 0.
+
+When setting `0`, the actual default is `available_processors * 2`
+
+This is an expert-level setting, and generally should not need to be set
+{plugin-uc} plugin is implemented based on a non-blocking mechanism, requiring a number of event loop and executor threads.
+The event loop threads are responsible to communicate with clients (accept incoming connections, enqueue/dequeue tasks and respond) and executor threads handle tasks.
+This configuration intends to limit or increase the number of threads to be created for the event loop.
+See <<plugins-{type}s-{plugin}-executor_threads>> configuration if you need to set executor threads count.
+
 [id="plugins-{type}s-{plugin}-executor_threads"]
 ===== `executor_threads`
 

data/lib/logstash/inputs/beats/message_listener.rb

@@ -9,6 +9,7 @@ module LogStash module Inputs class Beats
     include org.logstash.beats.IMessageListener
 
     FILEBEAT_LOG_LINE_FIELD = "message".freeze
+    LSF_LOG_LINE_FIELD = "line".freeze
 
     ConnectionState = Struct.new(:ctx, :codec, :ip_address)
 
@@ -182,6 +183,8 @@ module LogStash module Inputs class Beats
     def extract_target_field(hash)
       if from_filebeat?(hash)
         hash.delete(FILEBEAT_LOG_LINE_FIELD).to_s
+      elsif from_logstash_forwarder?(hash)
+        hash.delete(LSF_LOG_LINE_FIELD).to_s
       end
     end
 
@@ -189,6 +192,10 @@ module LogStash module Inputs class Beats
       !hash[FILEBEAT_LOG_LINE_FIELD].nil?
     end
 
+    def from_logstash_forwarder?(hash)
+      !hash[LSF_LOG_LINE_FIELD].nil?
+    end
+
     def increment_connection_count
       current_connection_count = @connections_list.size
       @metric.gauge(:current_connections, current_connection_count)

data/lib/logstash/inputs/beats.rb

@@ -141,6 +141,10 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   # Beats handler executor thread
   config :executor_threads, :validate => :number, :default => LogStash::Config::CpuCoreStrategy.maximum
 
+  # Expert only setting which set's Netty Event Loop Group thread count
+  # defaults to zero where Netty's DEFAULT_EVENT_LOOP_THREADS (NettyRuntime.availableProcessors() * 2) will be applied
+  config :event_loop_threads, :validate => :number, :default => 0
+
   # Flag to determine whether to add host information (provided by the beat in the 'hostname' field) to the event
   config :add_hostname, :validate => :boolean, :default => false, :deprecated => 'This option will be removed in the future as beats determine the event schema'
 
@@ -243,7 +247,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   end # def register
 
   def create_server
-    server = org.logstash.beats.Server.new(@host, @port, @client_inactivity_timeout, @executor_threads)
+    server = org.logstash.beats.Server.new(@host, @port, @client_inactivity_timeout, @event_loop_threads, @executor_threads)
     server.setSslHandlerProvider(new_ssl_handshake_provider(new_ssl_context_builder)) if @ssl_enabled
     server
   end

data/lib/logstash-input-beats_jars.rb

@@ -8,4 +8,4 @@ require_jar('io.netty', 'netty-transport', '4.1.100.Final')
 require_jar('io.netty', 'netty-handler', '4.1.100.Final')
 require_jar('io.netty', 'netty-transport-native-unix-common', '4.1.100.Final')
 require_jar('org.javassist', 'javassist', '3.24.0-GA')
-require_jar('org.logstash.beats', 'logstash-input-beats', '6.7.1')
+require_jar('org.logstash.beats', 'logstash-input-beats', '6.8.0')

data/lib/tasks/test.rake

@@ -9,6 +9,8 @@ elsif OS_PLATFORM == "darwin"
   FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-darwin-x86_64.tar.gz"
 end
 
+LSF_URL = "https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder_#{OS_PLATFORM}_amd64"
+
 require "fileutils"
 @files=[]
 
@@ -25,9 +27,21 @@ namespace :test do
   namespace :integration do
     task :setup do
       Rake::Task["test:integration:setup:filebeat"].invoke
+      Rake::Task["test:integration:setup:lsf"].invoke
     end
 
     namespace :setup do
+      desc "Download latest stable version of Logstash-forwarder"
+      task :lsf do
+        destination = File.join(VENDOR_PATH, "logstash-forwarder")
+        FileUtils.rm_rf(destination)
+        FileUtils.mkdir_p(destination)
+        download_destination = File.join(destination, "logstash-forwarder")
+        puts "Logstash-forwarder: downloading from #{LSF_URL} to #{download_destination}"
+        download(LSF_URL, download_destination)
+        File.chmod(0755, download_destination)
+      end
+
       desc "Download nigthly filebeat for integration testing"
       task :filebeat do
         FileUtils.mkdir_p(VENDOR_PATH)
@@ -46,7 +60,7 @@ namespace :test do
 end
 
 # Uncompress all the file from the archive this only work with 
-# one level directory structure and filebeat packaging.
+# one level directory structure and its fine for LSF and filebeat packaging.
 def untar_all(file, destination)
   untar(file) do |entry| 
     out = entry.full_name.split("/").last

data/spec/inputs/beats/message_listener_spec.rb

@@ -199,6 +199,16 @@ describe LogStash::Inputs::Beats::MessageListener do
       end
     end
 
+    context "when the message is from LSF" do
+      let(:message) { MockMessage.new("abc", { "line" => "hello world", '@metadata' => {} } )}
+
+      it "extract the event" do
+        subject.onNewMessage(ctx, message)
+        event = queue.pop
+        expect(event.get("message")).to eq("hello world")
+      end
+    end
+
     it_behaves_like "when the message is from any libbeat", :disabled, "[@metadata][ip_address]"
     it_behaves_like "when the message is from any libbeat", :v1, "[@metadata][input][beats][host][ip]"
     it_behaves_like "when the message is from any libbeat", :v8, "[@metadata][input][beats][host][ip]"

data/spec/inputs/beats_spec.rb

@@ -13,7 +13,8 @@ describe LogStash::Inputs::Beats do
   let(:certificate) { BeatsInputTest.certificate }
   let(:port) { BeatsInputTest.random_port }
   let(:client_inactivity_timeout) { 400 }
-  let(:threads) { 1 + rand(9) }
+  let(:event_loop_threads) { 1 + rand(4) }
+  let(:executor_threads) { 1 + rand(9) }
   let(:queue)  { Queue.new }
   let(:config)   do
     {
@@ -21,7 +22,8 @@ describe LogStash::Inputs::Beats do
         "ssl_certificate" => certificate.ssl_cert,
         "ssl_key" => certificate.ssl_key,
         "client_inactivity_timeout" => client_inactivity_timeout,
-        "executor_threads" => threads,
+        "event_loop_threads" => event_loop_threads,
+        "executor_threads" => executor_threads,
         "type" => "example",
         "tags" => "beats"
     }
@@ -36,7 +38,7 @@ describe LogStash::Inputs::Beats do
       let(:port) { 9001 }
 
       it "sends the required options to the server" do
-        expect(org.logstash.beats.Server).to receive(:new).with(host, port, client_inactivity_timeout, threads)
+        expect(org.logstash.beats.Server).to receive(:new).with(host, port, client_inactivity_timeout, event_loop_threads, executor_threads)
         subject.register
       end
     end
@@ -529,8 +531,8 @@ describe LogStash::Inputs::Beats do
     subject(:plugin) { LogStash::Inputs::Beats.new(config) }
 
     before do
-      @server = org.logstash.beats.Server.new(host, port, client_inactivity_timeout, threads)
-      expect( org.logstash.beats.Server ).to receive(:new).with(host, port, client_inactivity_timeout, threads).and_return @server
+      @server = org.logstash.beats.Server.new(host, port, client_inactivity_timeout, event_loop_threads, executor_threads)
+      expect( org.logstash.beats.Server ).to receive(:new).with(host, port, client_inactivity_timeout, event_loop_threads, executor_threads).and_return @server
       expect( @server ).to receive(:listen)
 
       subject.register

data/spec/integration/logstash_forwarder_spec.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "logstash/json"
+require "fileutils"
+require_relative "../support/flores_extensions"
+require_relative "../support/file_helpers"
+require_relative "../support/integration_shared_context"
+require_relative "../support/client_process_helpers"
+require "spec_helper"
+
+LSF_BINARY = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor", "logstash-forwarder", "logstash-forwarder"))
+
+describe "Logstash-Forwarder", :integration => true do
+  include ClientProcessHelpers
+
+  before :all do
+    unless File.exist?(LSF_BINARY)
+      raise "Cannot find `logstash-forwarder` binary in `vendor/logstash-forwarder` Did you run `bundle exec rake test:integration:setup` before running the integration suite?"
+    end
+  end
+
+  include FileHelpers
+  include_context "beats configuration"
+
+  let(:client_wait_time) { 5 }
+
+  # Filebeat related variables
+  let(:cmd) { [lsf_exec, "-config", lsf_config_path] }
+
+  let(:lsf_exec) { LSF_BINARY }
+  let(:registry_file) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", ".logstash-forwarder")) }
+  let_tmp_file(:lsf_config_path) { lsf_config }
+  let(:log_file) { f = Stud::Temporary.file; f.close; f.path }
+  let(:lsf_config) do
+ <<-CONFIG
+{
+  "network": {
+    "servers": [ "#{host}:#{port}" ],
+    "ssl ca": "#{certificate_authorities}"
+  },
+  "files": [
+    { "paths": [ "#{log_file}" ] }
+  ]
+}
+ CONFIG
+  end
+  #
+
+  before :each do
+    FileUtils.rm_rf(registry_file) # ensure clean state between runs
+    start_client
+    sleep(1)
+    # let LSF start and than write the logs
+    File.open(log_file, "a") do |f|
+      f.write(events.join("\n") + "\n")
+    end
+    sleep(1) # give some time to the clients to pick up the changes
+  end
+
+  after :each do
+    stop_client
+  end
+
+  xcontext "Plain TCP" do
+    include ClientProcessHelpers
+
+    let(:certificate_authorities) { "" }
+
+    it "should not send any events" do
+      expect(queue.size).to eq(0), @execution_output
+    end
+  end
+
+  context "TLS" do
+    context "Server Verification" do
+      let(:input_config) do
+        super().merge({
+          "ssl_enabled" => true,
+          "ssl_certificate" => certificate_file,
+          "ssl_key" => certificate_key_file,
+        })
+      end
+
+      let(:certificate_data) { Flores::PKI.generate }
+      let_tmp_file(:certificate_file) { certificate_data.first }
+      let_tmp_file(:certificate_key_file) { convert_to_pkcs8(certificate_data.last) }
+      let(:certificate_authorities) { certificate_file }
+
+      context "self signed certificate" do
+        include_examples "send events"
+      end
+
+      context "with large batches" do
+        let(:number_of_events) { 10_000 }
+        include_examples "send events"
+      end
+
+      context "invalid CA on the client" do
+        let(:invalid_data) { Flores::PKI.generate }
+        let(:certificate_authorities) { f = Stud::Temporary.file; f.close; f.path }
+
+        it "should not send any events" do
+          expect(queue.size).to eq(0), @execution_output
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 6.7.1
+  version: 6.8.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-18 00:00:00.000000000 Z
+date: 2024-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -311,6 +311,7 @@ files:
 - spec/inputs/beats/tls_spec.rb
 - spec/inputs/beats_spec.rb
 - spec/integration/filebeat_spec.rb
+- spec/integration/logstash_forwarder_spec.rb
 - spec/spec_helper.rb
 - spec/support/client_process_helpers.rb
 - spec/support/file_helpers.rb
@@ -326,7 +327,7 @@ files:
 - vendor/jar-dependencies/io/netty/netty-transport-native-unix-common/4.1.100.Final/netty-transport-native-unix-common-4.1.100.Final.jar
 - vendor/jar-dependencies/io/netty/netty-transport/4.1.100.Final/netty-transport-4.1.100.Final.jar
 - vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar
-- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.7.1/logstash-input-beats-6.7.1.jar
+- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.8.0/logstash-input-beats-6.8.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -362,6 +363,7 @@ test_files:
 - spec/inputs/beats/tls_spec.rb
 - spec/inputs/beats_spec.rb
 - spec/integration/filebeat_spec.rb
+- spec/integration/logstash_forwarder_spec.rb
 - spec/spec_helper.rb
 - spec/support/client_process_helpers.rb
 - spec/support/file_helpers.rb