logstash-input-beats 6.7.1-java → 6.7.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 633344379c679c34cba9b726bf6743f8dad7da7b65b4266e647d1539c3ef3a08
-  data.tar.gz: 340d1c62e61b64f6787a8e87df877cfd6073aa42ff9601504f21272119fc9937
+  metadata.gz: cfe0b8587cd80a4285628570381bf998f5765e4d779ee02173b18c90cea621cc
+  data.tar.gz: 9f7b15e3627e6b6a594b9e2227185a7875b321d0a7062639d06332047480a59e
 SHA512:
-  metadata.gz: 779e37e64ba62a4fd988a535866f393bb4797980eee02260b56ddf145194f398f1c2abfb5708d1939279bb4fcf2a6cba0b59b061ea59ea8c57d2a700d64e2e81
-  data.tar.gz: 39f78d631e00518b17af3d88109b3c41f58a3e9ef61d26e7e2a7c97eb0fb0c915b6f7d994b6975d5fd4d0f95cf5e24a54b7d959b373b501dacda8d639c19b04f
+  metadata.gz: 7f4f396f596f515f4194083e8e590a83a84e4d6f1e222f7a12de1a3dcb7483c830654f806781ab1fa494568f562fbfccfed09f0a40780858fcec70bf4a46c1e0
+  data.tar.gz: 1307df00e5d5edc2325020dbd8dd0f1902dcf4404412869f254a2b9303560267cebae254cf4852138533b8a6281ce599fb2d86312c2077792982338d9f471e3e

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 6.7.2
+  - Restore accidentally removed Lumberjack event parse source lines [#486](https://github.com/logstash-plugins/logstash-input-beats/pull/486)
+
 ## 6.7.1
   - bump netty to 4.1.100 and jackson to 2.15.3 [#484](https://github.com/logstash-plugins/logstash-input-beats/pull/484)
 

data/VERSION

@@ -1 +1 @@
-6.7.1
+6.7.2

data/lib/logstash/inputs/beats/message_listener.rb

@@ -9,6 +9,7 @@ module LogStash module Inputs class Beats
     include org.logstash.beats.IMessageListener
 
     FILEBEAT_LOG_LINE_FIELD = "message".freeze
+    LSF_LOG_LINE_FIELD = "line".freeze
 
     ConnectionState = Struct.new(:ctx, :codec, :ip_address)
 
@@ -182,6 +183,8 @@ module LogStash module Inputs class Beats
     def extract_target_field(hash)
       if from_filebeat?(hash)
         hash.delete(FILEBEAT_LOG_LINE_FIELD).to_s
+      elsif from_logstash_forwarder?(hash)
+        hash.delete(LSF_LOG_LINE_FIELD).to_s
       end
     end
 
@@ -189,6 +192,10 @@ module LogStash module Inputs class Beats
       !hash[FILEBEAT_LOG_LINE_FIELD].nil?
     end
 
+    def from_logstash_forwarder?(hash)
+      !hash[LSF_LOG_LINE_FIELD].nil?
+    end
+
     def increment_connection_count
       current_connection_count = @connections_list.size
       @metric.gauge(:current_connections, current_connection_count)

data/lib/logstash-input-beats_jars.rb

@@ -8,4 +8,4 @@ require_jar('io.netty', 'netty-transport', '4.1.100.Final')
 require_jar('io.netty', 'netty-handler', '4.1.100.Final')
 require_jar('io.netty', 'netty-transport-native-unix-common', '4.1.100.Final')
 require_jar('org.javassist', 'javassist', '3.24.0-GA')
-require_jar('org.logstash.beats', 'logstash-input-beats', '6.7.1')
+require_jar('org.logstash.beats', 'logstash-input-beats', '6.7.2')

data/lib/tasks/test.rake

@@ -9,6 +9,8 @@ elsif OS_PLATFORM == "darwin"
   FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-darwin-x86_64.tar.gz"
 end
 
+LSF_URL = "https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder_#{OS_PLATFORM}_amd64"
+
 require "fileutils"
 @files=[]
 
@@ -25,9 +27,21 @@ namespace :test do
   namespace :integration do
     task :setup do
       Rake::Task["test:integration:setup:filebeat"].invoke
+      Rake::Task["test:integration:setup:lsf"].invoke
     end
 
     namespace :setup do
+      desc "Download latest stable version of Logstash-forwarder"
+      task :lsf do
+        destination = File.join(VENDOR_PATH, "logstash-forwarder")
+        FileUtils.rm_rf(destination)
+        FileUtils.mkdir_p(destination)
+        download_destination = File.join(destination, "logstash-forwarder")
+        puts "Logstash-forwarder: downloading from #{LSF_URL} to #{download_destination}"
+        download(LSF_URL, download_destination)
+        File.chmod(0755, download_destination)
+      end
+
       desc "Download nigthly filebeat for integration testing"
       task :filebeat do
         FileUtils.mkdir_p(VENDOR_PATH)
@@ -46,7 +60,7 @@ namespace :test do
 end
 
 # Uncompress all the file from the archive this only work with 
-# one level directory structure and filebeat packaging.
+# one level directory structure and its fine for LSF and filebeat packaging.
 def untar_all(file, destination)
   untar(file) do |entry| 
     out = entry.full_name.split("/").last

data/spec/inputs/beats/message_listener_spec.rb

@@ -199,6 +199,16 @@ describe LogStash::Inputs::Beats::MessageListener do
       end
     end
 
+    context "when the message is from LSF" do
+      let(:message) { MockMessage.new("abc", { "line" => "hello world", '@metadata' => {} } )}
+
+      it "extract the event" do
+        subject.onNewMessage(ctx, message)
+        event = queue.pop
+        expect(event.get("message")).to eq("hello world")
+      end
+    end
+
     it_behaves_like "when the message is from any libbeat", :disabled, "[@metadata][ip_address]"
     it_behaves_like "when the message is from any libbeat", :v1, "[@metadata][input][beats][host][ip]"
     it_behaves_like "when the message is from any libbeat", :v8, "[@metadata][input][beats][host][ip]"

data/spec/integration/logstash_forwarder_spec.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "logstash/json"
+require "fileutils"
+require_relative "../support/flores_extensions"
+require_relative "../support/file_helpers"
+require_relative "../support/integration_shared_context"
+require_relative "../support/client_process_helpers"
+require "spec_helper"
+
+LSF_BINARY = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor", "logstash-forwarder", "logstash-forwarder"))
+
+describe "Logstash-Forwarder", :integration => true do
+  include ClientProcessHelpers
+
+  before :all do
+    unless File.exist?(LSF_BINARY)
+      raise "Cannot find `logstash-forwarder` binary in `vendor/logstash-forwarder` Did you run `bundle exec rake test:integration:setup` before running the integration suite?"
+    end
+  end
+
+  include FileHelpers
+  include_context "beats configuration"
+
+  let(:client_wait_time) { 5 }
+
+  # Filebeat related variables
+  let(:cmd) { [lsf_exec, "-config", lsf_config_path] }
+
+  let(:lsf_exec) { LSF_BINARY }
+  let(:registry_file) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", ".logstash-forwarder")) }
+  let_tmp_file(:lsf_config_path) { lsf_config }
+  let(:log_file) { f = Stud::Temporary.file; f.close; f.path }
+  let(:lsf_config) do
+ <<-CONFIG
+{
+  "network": {
+    "servers": [ "#{host}:#{port}" ],
+    "ssl ca": "#{certificate_authorities}"
+  },
+  "files": [
+    { "paths": [ "#{log_file}" ] }
+  ]
+}
+ CONFIG
+  end
+  #
+
+  before :each do
+    FileUtils.rm_rf(registry_file) # ensure clean state between runs
+    start_client
+    sleep(1)
+    # let LSF start and than write the logs
+    File.open(log_file, "a") do |f|
+      f.write(events.join("\n") + "\n")
+    end
+    sleep(1) # give some time to the clients to pick up the changes
+  end
+
+  after :each do
+    stop_client
+  end
+
+  xcontext "Plain TCP" do
+    include ClientProcessHelpers
+
+    let(:certificate_authorities) { "" }
+
+    it "should not send any events" do
+      expect(queue.size).to eq(0), @execution_output
+    end
+  end
+
+  context "TLS" do
+    context "Server Verification" do
+      let(:input_config) do
+        super().merge({
+          "ssl_enabled" => true,
+          "ssl_certificate" => certificate_file,
+          "ssl_key" => certificate_key_file,
+        })
+      end
+
+      let(:certificate_data) { Flores::PKI.generate }
+      let_tmp_file(:certificate_file) { certificate_data.first }
+      let_tmp_file(:certificate_key_file) { convert_to_pkcs8(certificate_data.last) }
+      let(:certificate_authorities) { certificate_file }
+
+      context "self signed certificate" do
+        include_examples "send events"
+      end
+
+      context "with large batches" do
+        let(:number_of_events) { 10_000 }
+        include_examples "send events"
+      end
+
+      context "invalid CA on the client" do
+        let(:invalid_data) { Flores::PKI.generate }
+        let(:certificate_authorities) { f = Stud::Temporary.file; f.close; f.path }
+
+        it "should not send any events" do
+          expect(queue.size).to eq(0), @execution_output
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 6.7.1
+  version: 6.7.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-18 00:00:00.000000000 Z
+date: 2023-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -311,6 +311,7 @@ files:
 - spec/inputs/beats/tls_spec.rb
 - spec/inputs/beats_spec.rb
 - spec/integration/filebeat_spec.rb
+- spec/integration/logstash_forwarder_spec.rb
 - spec/spec_helper.rb
 - spec/support/client_process_helpers.rb
 - spec/support/file_helpers.rb
@@ -326,7 +327,7 @@ files:
 - vendor/jar-dependencies/io/netty/netty-transport-native-unix-common/4.1.100.Final/netty-transport-native-unix-common-4.1.100.Final.jar
 - vendor/jar-dependencies/io/netty/netty-transport/4.1.100.Final/netty-transport-4.1.100.Final.jar
 - vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar
-- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.7.1/logstash-input-beats-6.7.1.jar
+- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.7.2/logstash-input-beats-6.7.2.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -362,6 +363,7 @@ test_files:
 - spec/inputs/beats/tls_spec.rb
 - spec/inputs/beats_spec.rb
 - spec/integration/filebeat_spec.rb
+- spec/integration/logstash_forwarder_spec.rb
 - spec/spec_helper.rb
 - spec/support/client_process_helpers.rb
 - spec/support/file_helpers.rb