logstash-input-beats 6.0.11-java → 6.1.1-java

data/lib/logstash/inputs/beats/decoded_event_transform.rb

@@ -9,7 +9,16 @@ module LogStash module Inputs class Beats
       ts = coerce_ts(hash.delete("@timestamp"))
 
       event.set("@timestamp", ts) unless ts.nil?
-      hash.each { |k, v| event.set(k, v) }
+      hash.each do |k, v|
+        #could be a nested map, so we need to merge and not overwrite
+        existing_value = event.get(k)
+        if existing_value.is_a?(Hash)
+          existing_value = existing_value.merge(v)
+        else
+          existing_value = v
+        end
+        event.set(k, existing_value)
+      end
       super(event)
       event.tag("beats_input_codec_#{codec_name}_applied") if include_codec_tag?
       event

data/lib/logstash/inputs/beats/event_transform_common.rb

@@ -15,8 +15,8 @@ module LogStash module Inputs class Beats
       return unless @input.add_hostname
       host = event.get("[beat][hostname]")
 
-      if host && event.get("host").nil?
-        event.set("host", host)
+      if host && event.get(@input.field_hostname).nil?
+        event.set(@input.field_hostname, host)
       end
     end
 

data/lib/logstash/inputs/beats/message_listener.rb

@@ -31,7 +31,9 @@ module LogStash module Inputs class Beats
       hash = message.getData
       ip_address = ip_address(ctx)
 
-      hash['@metadata']['ip_address'] = ip_address unless ip_address.nil? || hash['@metadata'].nil?
+      unless ip_address.nil? || hash['@metadata'].nil?
+        set_nested(hash, @input.field_hostip, ip_address)
+      end
       target_field = extract_target_field(hash)
 
       extract_tls_peer(hash, ctx)
@@ -140,11 +142,12 @@ module LogStash module Inputs class Beats
         end
 
         if tls_verified
+          set_nested(hash, @field_tls_protocol_version, tls_session.getProtocol())
+          set_nested(hash, @field_tls_peer_subject, tls_session.getPeerPrincipal().getName())
+          set_nested(hash, @field_tls_cipher, tls_session.getCipherSuite())
+
           hash['@metadata']['tls_peer'] = {
-            :status       => "verified",
-            :protocol     => tls_session.getProtocol(),
-            :subject      => tls_session.getPeerPrincipal().getName(),
-            :cipher_suite => tls_session.getCipherSuite()
+            :status       => "verified"
           }
         else
           hash['@metadata']['tls_peer'] = {
@@ -154,6 +157,28 @@ module LogStash module Inputs class Beats
       end
     end
 
+    # set the value for field_name into the hash, nesting into sub-hashes and creating hashes where necessary
+    public #only to make it testable
+    def set_nested(hash, field_name, value)
+      field_ref = Java::OrgLogstash::FieldReference.from(field_name)
+      # create @metadata sub-hash if needed
+      if field_ref.type == Java::OrgLogstash::FieldReference::META_CHILD
+        unless hash.key?("@metadata")
+          hash["@metadata"] = {}
+        end
+        nesting_hash = hash["@metadata"]
+      else
+        nesting_hash = hash
+      end
+
+      field_ref.path.each do |token|
+        nesting_hash[token] = {} unless nesting_hash.key?(token)
+        nesting_hash = nesting_hash[token]
+      end
+      nesting_hash[field_ref.key] = value
+    end
+
+    private
     def extract_target_field(hash)
       if from_filebeat?(hash)
         hash.delete(FILEBEAT_LOG_LINE_FIELD).to_s

data/logstash-input-beats.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "thread_safe", "~> 0.3.5"
   s.add_runtime_dependency "logstash-codec-multiline", ">= 2.0.5"
   s.add_runtime_dependency 'jar-dependencies', '~> 0.3', '>= 0.3.4'
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
 
   s.add_development_dependency "flores", "~>0.0.6"
   s.add_development_dependency "rspec"

data/spec/inputs/beats/decoded_event_transform_spec.rb

@@ -29,7 +29,8 @@ describe LogStash::Inputs::Beats::DecodedEventTransform do
 
   subject { described_class.new(input).transform(event, map) }
 
-  include_examples "Common Event Transformation"
+  include_examples "Common Event Transformation", :disabled, "host"
+  include_examples "Common Event Transformation", :v1, "[@metadata][input][beats][host][name]"
 
   it "tags the event" do
     expect(subject.get("tags")).to include("beats_input_codec_plain_applied")
@@ -43,7 +44,7 @@ describe LogStash::Inputs::Beats::DecodedEventTransform do
   context "map contains a timestamp" do
     context "when its valid" do
       let(:timestamp) { Time.now }
-      let(:map) { super.merge({"@timestamp" => timestamp }) }
+      let(:map) { super().merge({"@timestamp" => timestamp }) }
      
       it "uses as the event timestamp" do
         expect(subject.get("@timestamp")).to eq(LogStash::Timestamp.coerce(timestamp)) 
@@ -51,7 +52,7 @@ describe LogStash::Inputs::Beats::DecodedEventTransform do
     end
 
     context "when its not valid" do
-      let(:map) { super.merge({"@timestamp" => "invalid" }) }
+      let(:map) { super().merge({"@timestamp" => "invalid" }) }
 
       it "fallback the current time" do
         expect(subject.get("@timestamp")).to be_kind_of(LogStash::Timestamp)

data/spec/inputs/beats/event_transform_common_spec.rb

@@ -7,5 +7,6 @@ require "spec_helper"
 describe LogStash::Inputs::Beats::EventTransformCommon do
   subject { described_class.new(input).transform(event) }
 
-  include_examples "Common Event Transformation"
+  include_examples "Common Event Transformation", :disabled, "host"
+  include_examples "Common Event Transformation", :v1, "[@metadata][input][beats][host][name]"
 end

data/spec/inputs/beats/message_listener_spec.rb

@@ -52,10 +52,73 @@ class DummyCodec < LogStash::Codecs::Base
   end
 end
 
+shared_examples "when the message is from any libbeat" do |ecs_compatibility, host_field_name|
+  let(:input) do
+    input = LogStash::Inputs::Beats.new({ "port" => 5555, "codec" => codec, "ecs_compatibility" => "#{ecs_compatibility}" })
+    input.register
+    input
+  end
+
+  #Requires data modeled as Java, not Ruby since the actual code pulls from Java backed (Netty) object
+  let(:data) do
+    d = HashMap.new
+    d.put('@metadata', HashMap.new)
+    d.put('metric',  1)
+    d.put('name', "super-stats")
+    d
+  end
+
+  let(:message) { MockMessage.new("abc",  data)}
+
+  it "extract the event" do
+    subject.onNewMessage(ctx, message)
+    event = queue.pop
+    expect(event.get("message")).to be_nil
+    expect(event.get("metric")).to eq(1)
+    expect(event.get("name")).to eq("super-stats")
+    expect(event.get(host_field_name)).to eq(ip_address)
+  end
+
+  context 'when the remote address is nil' do
+    let(:ctx) { OngoingMethodMock.new("remoteAddress", nil)}
+
+    it 'extracts the event' do
+      subject.onNewMessage(ctx, message)
+      event = queue.pop
+      expect(event.get("message")).to be_nil
+      expect(event.get("metric")).to eq(1)
+      expect(event.get("name")).to eq("super-stats")
+      expect(event.get(host_field_name)).to eq(nil)
+    end
+  end
+
+  context 'when getting the remote address raises' do
+    let(:raising_ctx) { double("context")}
+
+    before  do
+      allow(raising_ctx).to receive(:channel).and_raise("nope")
+      subject.onNewConnection(raising_ctx)
+    end
+
+    it 'extracts the event' do
+      subject.onNewMessage(raising_ctx, message)
+      event = queue.pop
+      expect(event.get("message")).to be_nil
+      expect(event.get("metric")).to eq(1)
+      expect(event.get("name")).to eq("super-stats")
+      expect(event.get(host_field_name)).to eq(nil)
+    end
+  end
+end
+
 describe LogStash::Inputs::Beats::MessageListener do
   let(:queue)  { Queue.new }
   let(:codec) { DummyCodec.new }
-  let(:input) { LogStash::Inputs::Beats.new({ "port" => 5555, "codec" => codec }) }
+  let(:input) do
+    input = LogStash::Inputs::Beats.new({ "port" => 5555, "codec" => codec })
+    input.register
+    input
+  end
 
   let(:ip_address) { "10.0.0.1" }
   let(:remote_address) { OngoingMethodMock.new("getHostAddress", ip_address) }
@@ -146,59 +209,8 @@ describe LogStash::Inputs::Beats::MessageListener do
       end
     end
 
-    context "when the message is from any libbeat" do
-      #Requires data modeled as Java, not Ruby since the actual code pulls from Java backed (Netty) object
-      let(:data) do
-        d = HashMap.new
-        d.put('@metadata', HashMap.new)
-        d.put('metric',  1)
-        d.put('name', "super-stats")
-        d
-      end
-
-      let(:message) { MockMessage.new("abc",  data)}
-
-      it "extract the event" do
-        subject.onNewMessage(ctx, message)
-        event = queue.pop
-        expect(event.get("message")).to be_nil
-        expect(event.get("metric")).to eq(1)
-        expect(event.get("name")).to eq("super-stats")
-        expect(event.get("[@metadata][ip_address]")).to eq(ip_address)
-      end
-
-      context 'when the remote address is nil' do
-        let(:ctx) { OngoingMethodMock.new("remoteAddress", nil)}
-
-        it 'extracts the event' do
-          subject.onNewMessage(ctx, message)
-          event = queue.pop
-          expect(event.get("message")).to be_nil
-          expect(event.get("metric")).to eq(1)
-          expect(event.get("name")).to eq("super-stats")
-          expect(event.get("[@metadata][ip_address]")).to eq(nil)
-        end
-      end
-
-      context 'when getting the remote address raises' do
-        let(:raising_ctx) { double("context")}
-
-        before  do
-          allow(raising_ctx).to receive(:channel).and_raise("nope")
-          subject.onNewConnection(raising_ctx)
-        end
-
-        it 'extracts the event' do
-          subject.onNewMessage(raising_ctx, message)
-          event = queue.pop
-          expect(event.get("message")).to be_nil
-          expect(event.get("metric")).to eq(1)
-          expect(event.get("name")).to eq("super-stats")
-          expect(event.get("[@metadata][ip_address]")).to eq(nil)
-        end
-      end
-
-    end
+    it_behaves_like "when the message is from any libbeat", :disabled, "[@metadata][ip_address]"
+    it_behaves_like "when the message is from any libbeat", :v1, "[@metadata][input][beats][host][ip]"
   end
 
   context "onException" do
@@ -222,4 +234,20 @@ describe LogStash::Inputs::Beats::MessageListener do
       expect(queue).not_to be_empty
     end
   end
+
+  context "set_nested" do
+    let(:hash) {{}}
+
+    it "creates correctly the nested maps" do
+      subject.set_nested(hash, "[root][inner][leaf]", 5)
+      expect(hash["root"]["inner"]["leaf"]).to eq(5)
+    end
+
+    it "doesn't overwrite existing the nested maps" do
+      hash = {"root" => {"foo" => {"bar" => "Hello"}}}
+      subject.set_nested(hash, "[root][inner][leaf]", 5)
+      expect(hash["root"]["inner"]["leaf"]).to eq(5)
+      expect(hash["root"]["foo"]["bar"]).to eq("Hello")
+    end
+  end
 end

data/spec/inputs/beats/raw_event_transform_spec.rb

@@ -18,7 +18,8 @@ describe LogStash::Inputs::Beats::RawEventTransform do
 
   subject { described_class.new(input).transform(event) }
 
-  include_examples "Common Event Transformation"
+  include_examples "Common Event Transformation", :disabled, "host"
+  include_examples "Common Event Transformation", :v1, "[@metadata][input][beats][host][name]"
 
   it "tags the event" do
     expect(subject.get("tags")).to include("beats_input_raw_event")

data/spec/inputs/beats_spec.rb

@@ -13,11 +13,19 @@ describe LogStash::Inputs::Beats do
   let(:certificate) { BeatsInputTest.certificate }
   let(:port) { BeatsInputTest.random_port }
   let(:queue)  { Queue.new }
-  let(:config)   { { "port" => 0, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "beats"} }
+  let(:config)   do
+    {
+        "port" => 0,
+        "ssl_certificate" => certificate.ssl_cert,
+        "ssl_key" => certificate.ssl_key,
+        "type" => "example",
+        "tags" => "beats"
+    }
+  end
 
   context "#register" do
     context "host related configuration" do
-      let(:config) { super.merge!({ "host" => host, "port" => port, "client_inactivity_timeout" => client_inactivity_timeout, "executor_threads" => threads }) }
+      let(:config) { super().merge("host" => host, "port" => port, "client_inactivity_timeout" => client_inactivity_timeout, "executor_threads" => threads) }
       let(:host) { "192.168.1.20" }
       let(:port) { 9000 }
       let(:client_inactivity_timeout) { 400 }
@@ -38,38 +46,55 @@ describe LogStash::Inputs::Beats do
 
     context "with ssl enabled" do
       context "without certificate configuration" do
-        let(:config) {{ "port" => 0, "ssl" => true, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "beats" }}
+        let(:config) { { "port" => 0, "ssl" => true, "ssl_key" => certificate.ssl_key, "type" => "example" } }
 
         it "should fail to register the plugin with ConfigurationError" do
           plugin = LogStash::Inputs::Beats.new(config)
-          expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+          expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
         end
       end
 
       context "without key configuration" do
-        let(:config)   { { "port" => 0, "ssl" => true, "ssl_certificate" => certificate.ssl_cert, "type" => "example", "tags" => "Beats"} }
+        let(:config) { { "port" => 0, "ssl" => true, "ssl_certificate" => certificate.ssl_cert, "type" => "example" } }
         it "should fail to register the plugin with ConfigurationError" do
           plugin = LogStash::Inputs::Beats.new(config)
-          expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+          expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+        end
+      end
+
+      context "with invalid key configuration" do
+        let(:p12_key) { certificate.p12_key }
+        let(:config) { { "port" => 0, "ssl" => true, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => p12_key } }
+        it "should fail to register the plugin" do
+          plugin = LogStash::Inputs::Beats.new(config)
+          expect( plugin.logger ).to receive(:error) do |msg, opts|
+            expect( msg ).to match /.*?configuration invalid/
+            expect( opts[:message] ).to match /does not contain valid private key/
+          end
+          expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
         end
       end
 
       context "with invalid ciphers" do
-        let(:config)   { { "port" => 0, "ssl" => true, "ssl_certificate" => certificate.ssl_cert, "type" => "example", "tags" => "Beats", "cipher_suites" => "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA38"} }
+        let(:config) { super().merge("ssl" => true, "cipher_suites" => "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA38") }
 
         it "should raise a configuration error" do
           plugin = LogStash::Inputs::Beats.new(config)
+          expect( plugin.logger ).to receive(:error) do |msg, opts|
+            expect( msg ).to match /.*?configuration invalid/
+            expect( opts[:message] ).to match /TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA38.*? not available/
+          end
           expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
         end
       end
 
       context "verify_mode" do
         context "verify_mode configured to PEER" do
-          let(:config)   { { "port" => 0, "ssl" => true, "ssl_verify_mode" => "peer", "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "Beats"} }
+          let(:config) { super().merge("ssl" => true, "ssl_verify_mode" => "peer") }
 
           it "raise a ConfigurationError when certificate_authorities is not set" do
             plugin = LogStash::Inputs::Beats.new(config)
-            expect {plugin.register}.to raise_error(LogStash::ConfigurationError, "Using `verify_mode` set to PEER or FORCE_PEER, requires the configuration of `certificate_authorities`")
+            expect {plugin.register}.to raise_error(LogStash::ConfigurationError, "ssl_certificate_authorities => is a required setting when ssl_verify_mode => 'peer' is configured")
           end
 
           it "doesn't raise a configuration error when certificate_authorities is set" do
@@ -80,11 +105,11 @@ describe LogStash::Inputs::Beats do
         end
 
         context "verify_mode configured to FORCE_PEER" do
-          let(:config)   { { "port" => 0, "ssl" => true, "ssl_verify_mode" => "force_peer", "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "Beats"} }
+          let(:config) { super().merge("ssl" => true, "ssl_verify_mode" => "force_peer") }
 
           it "raise a ConfigurationError when certificate_authorities is not set" do
             plugin = LogStash::Inputs::Beats.new(config)
-            expect {plugin.register}.to raise_error(LogStash::ConfigurationError, "Using `verify_mode` set to PEER or FORCE_PEER, requires the configuration of `certificate_authorities`")
+            expect {plugin.register}.to raise_error(LogStash::ConfigurationError, "ssl_certificate_authorities => is a required setting when ssl_verify_mode => 'force_peer' is configured")
           end
 
           it "doesn't raise a configuration error when certificate_authorities is set" do
@@ -98,7 +123,7 @@ describe LogStash::Inputs::Beats do
 
     context "with ssl disabled" do
       context "and certificate configuration" do
-        let(:config)   { { "port" => 0, "ssl" => false, "ssl_certificate" => certificate.ssl_cert, "type" => "example", "tags" => "Beats" } }
+        let(:config) { { "port" => 0, "ssl" => false, "ssl_certificate" => certificate.ssl_cert, "type" => "example", "tags" => "Beats" } }
 
         it "should not fail" do
           plugin = LogStash::Inputs::Beats.new(config)
@@ -129,7 +154,7 @@ describe LogStash::Inputs::Beats do
       let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '^2015',
                                                     "what" => "previous",
                                                     "negate" => true) }
-      let(:config) { super.merge({ "codec" => codec }) }
+      let(:config) { super().merge({ "codec" => codec }) }
 
       it "raise a ConfigurationError when multiline codec is set" do
         plugin = LogStash::Inputs::Beats.new(config)

data/spec/integration/filebeat_spec.rb

@@ -86,7 +86,7 @@ describe "Filebeat", :integration => true do
     end
 
     context "without pipelining" do
-      let(:filebeat_config) { config = super; config["output"]["logstash"]["pipelining"] = 0; config }
+      let(:filebeat_config) { config = super(); config["output"]["logstash"]["pipelining"] = 0; config }
       include_examples "send events"
 
       context "with large batches" do
@@ -99,7 +99,7 @@ describe "Filebeat", :integration => true do
   context "TLS" do
     context "Server verification" do
       let(:filebeat_config) do
-        super.merge({
+        super().merge({
           "output" => {
             "logstash" => {
               "hosts" => ["#{host}:#{port}"],
@@ -111,7 +111,7 @@ describe "Filebeat", :integration => true do
       end
 
       let(:input_config) do
-        super.merge({
+        super().merge({
           "ssl" => true,
           "ssl_certificate" => certificate_file,
           "ssl_key" => certificate_key_file
@@ -129,7 +129,7 @@ describe "Filebeat", :integration => true do
 
         context "when specifying a cipher" do
           let(:filebeat_config) do
-            super.merge({
+            super().merge({
               "output" => {
                 "logstash" => {
                   "hosts" => ["#{host}:#{port}"],
@@ -145,7 +145,7 @@ describe "Filebeat", :integration => true do
           end
 
           let(:input_config) {
-            super.merge({
+            super().merge({
               "cipher_suites" => [logstash_cipher],
               "tls_min_version" => "1.2"
             })
@@ -194,7 +194,7 @@ describe "Filebeat", :integration => true do
             LogStash::Inputs::Beats.new(input_config)
           }
           let(:input_config) {
-            super.merge({
+            super().merge({
             "ssl_key_passphrase" => passphrase,
             "ssl_key" => certificate_key_file_pkcs8
           })}
@@ -229,7 +229,7 @@ describe "Filebeat", :integration => true do
 
       context "Client verification / Mutual validation" do
         let(:filebeat_config) do
-          super.merge({
+          super().merge({
             "output" => {
               "logstash" => {
                 "hosts" => ["#{host}:#{port}"],
@@ -245,7 +245,7 @@ describe "Filebeat", :integration => true do
         end
 
         let(:input_config) do
-          super.merge({
+          super().merge({
             "ssl" => true,
             "ssl_certificate_authorities" => certificate_authorities,
             "ssl_certificate" => server_certificate_file,
@@ -327,7 +327,7 @@ describe "Filebeat", :integration => true do
 
               context "client from secondary CA" do
                 let(:filebeat_config) do
-                  super.merge({
+                  super().merge({
                     "output" => {
                       "logstash" => {
                         "hosts" => ["#{host}:#{port}"],