logstash-input-beats 5.0.4-java → 5.0.5-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a6d845ad2b35a5695019c5e3735f993c4ad4f30a75f744a15535e62d839ff4b
-  data.tar.gz: de65264eb05bfbca07af92c2aeb35ac2c49e795b728d788632b375ddcfd3c068
+  metadata.gz: c8556139762c9123ccc0204ec083924778a39f246ac992dd5d9e477e98068e72
+  data.tar.gz: b30065ad1d79a5020baae07c8b8a980a76a82bc84fadaccce4a54af667859d17
 SHA512:
-  metadata.gz: 2e6d24024876d7a5fd85f3bc717c5b534328bc2d05fe3ad13183db037635970b3dd11c0d7a3ada0d145daae8894f6332d9dcee42852576b1dde9bf9607cad2f8
-  data.tar.gz: 0da3bc63f17a30bd03eb1d85dec46f81262deeb98d89be0ffbf16457c99f1936961453aec6236583d90daddfa1e83186b2fa55b45e0b126fa6b7f30db804021e
+  metadata.gz: 01fcf5bae4b075a0ea5c58905a77c712f3ec317ad5d27417c75749a33f590b5572f0a9fa0fbbdd29618537f2a4a565edd51802efb4d3fe7c3b3225e3746cbbc2
+  data.tar.gz: 2ba8917e5b5576c0166e6a18cdc6677bd47daaa9abb282386c736fd83de0e7eef1addaa95c37da43a9891761384bd176f80a577c3bc6616aebad1e41e7c759d0

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.0.5
+  - Better handle case when remoteAddress is nil to reduce amount of warning messages in logs #269
+  
 ## 5.0.4
   - Fix an issue with `close_wait` connection and making sure the keep alive are send back to the client all the time. #272
 

data/VERSION

@@ -1 +1 @@
-5.0.4
+5.0.5

data/lib/logstash-input-beats_jars.rb

@@ -9,4 +9,4 @@ require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.7.5')
 require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.7.5')
 require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.7.5')
 require_jar('org.apache.logging.log4j', 'log4j-api', '2.6.2')
-require_jar('org.logstash.beats', 'logstash-input-beats', '5.0.4')
+require_jar('org.logstash.beats', 'logstash-input-beats', '5.0.5')

data/lib/logstash/inputs/beats/message_listener.rb

@@ -10,7 +10,7 @@ module LogStash module Inputs class Beats
     FILEBEAT_LOG_LINE_FIELD = "message".freeze
     LSF_LOG_LINE_FIELD = "line".freeze
 
-    ConnectionState = Struct.new(:ctx, :codec)
+    ConnectionState = Struct.new(:ctx, :codec, :ip_address)
 
     attr_reader :logger, :input, :connections_list
 
@@ -27,14 +27,10 @@ module LogStash module Inputs class Beats
     end
 
     def onNewMessage(ctx, message)
-      hash = message.getData()
-
-      begin
-        hash.get("@metadata").put("ip_address", ctx.channel().remoteAddress().getAddress().getHostAddress())
-      rescue #should never happen, but don't allow an error here to stop beats input
-        input.logger.warn("Could not retrieve remote IP address for beats input.")
-      end
+      hash = message.getData
+      ip_address = ip_address(ctx)
 
+      hash['@metadata']['ip_address'] = ip_address unless ip_address.nil? || hash['@metadata'].nil?
       target_field = extract_target_field(hash)
 
       if target_field.nil?
@@ -83,8 +79,29 @@ module LogStash module Inputs class Beats
       connections_list[ctx].codec
     end
 
+    def ip_address(ctx)
+      return if connections_list[ctx].nil?
+      connections_list[ctx].ip_address
+    end
+
     def register_connection(ctx)
-      connections_list[ctx] = ConnectionState.new(ctx, input.codec.dup)
+      connections_list[ctx] = ConnectionState.new(ctx, input.codec.dup, ip_address_from_ctx(ctx))
+    end
+
+    def ip_address_from_ctx(ctx)
+      begin
+        remote_address = ctx.channel.remoteAddress
+        # Netty allows remoteAddress to be nil, which can cause a lot of log entries - see
+        # https://github.com/logstash-plugins/logstash-input-beats/issues/269
+        if remote_address.nil?
+          input.logger.debug("Cannot retrieve remote IP address for beats input - remoteAddress is nil")
+          return nil
+        end
+        remote_address.getAddress.getHostAddress
+      rescue => e # This should not happen, but should not block the beats input
+        input.logger.warn("Could not retrieve remote IP address for beats input.", :error => e)
+        nil
+      end
     end
 
     def unregister_connection(ctx)

data/spec/inputs/beats/message_listener_spec.rb

@@ -56,7 +56,11 @@ describe LogStash::Inputs::Beats::MessageListener do
   let(:queue)  { Queue.new }
   let(:codec) { DummyCodec.new }
   let(:input) { LogStash::Inputs::Beats.new({ "port" => 5555, "codec" => codec }) }
-  let(:ctx) { double("ChannelHandlerContext") }
+
+  let(:ip_address) { "10.0.0.1" }
+  let(:remote_address) { OngoingMethodMock.new("getHostAddress", ip_address) }
+  let(:ctx) {OngoingMethodMock.new("remoteAddress", remote_address)}
+
   let(:message) { MockMessage.new("abc", { "message" => "hello world"}) }
 
   subject { described_class.new(queue, input) }
@@ -66,28 +70,54 @@ describe LogStash::Inputs::Beats::MessageListener do
   end
 
   context "onNewConnection" do
-    it "register the connection to the connection list" do
-      expect { subject.onNewConnection(double("ChannelHandlerContext")) }.to change { subject.connections_list.count }.by(1)
+    let(:second_ip_address) { "10.0.0.2" }
+    let(:second_ctx) {OngoingMethodMock.new("getHostAddress", second_ip_address)}
+
+    shared_examples 'a valid new connection' do
+      it "register the connection to the connection list" do
+        expect { subject.onNewConnection(second_ctx) }.to change { subject.connections_list.count }.by(1)
+      end
+    end
+
+    it_behaves_like 'a valid new connection'
+
+    context 'with a nil remote address' do
+      let(:second_ip_address) { nil}
+
+      it_behaves_like 'a valid new connection'
+    end
+
+    context 'when the channel throws retrieving remote address' do
+      before do
+        allow(second_ctx).to receive(:channel).and_raise('nope')
+      end
+
+      it_behaves_like 'a valid new connection'
     end
 
     describe "metrics" do
       it "new connection should increment connection count" do
         expect(subject).to receive(:increment_connection_count).once
-        subject.onNewConnection(double("ChannelHandlerContext"))
+        subject.onNewConnection(second_ctx)
       end
 
       describe "peak connections" do
+        let (:ctxes) { [1, 2, 3, 4].inject([]) do |result, element|
+          result << OngoingMethodMock.new("getHostAddress", "10.0.0.#{element}")
+          result
+        end
+        }
         it "closing and open connections should keep highest count" do
           expect(subject.instance_eval("@peak_connection_count").value).to eq(1)
-          subject.onNewConnection(1)
+          subject.onNewConnection(ctxes[0])
           expect(subject.instance_eval("@peak_connection_count").value).to eq(2)
-          subject.onNewConnection(2)
+          subject.onNewConnection(ctxes[1])
           expect(subject.instance_eval("@peak_connection_count").value).to eq(3)
-          subject.onConnectionClose(2)
+          subject.onConnectionClose(ctxes[1])
           expect(subject.instance_eval("@peak_connection_count").value).to eq(3)
-          subject.onNewConnection(3)
+          subject.onNewConnection(ctxes[2])
           expect(subject.instance_eval("@peak_connection_count").value).to eq(3)
-          subject.onNewConnection(4)
+          subject.onNewConnection(ctxes[3])
           expect(subject.instance_eval("@peak_connection_count").value).to eq(4)
         end
       end
@@ -97,7 +127,7 @@ describe LogStash::Inputs::Beats::MessageListener do
 
   context "onNewMessage" do
     context "when the message is from filebeat" do
-      let(:message) { MockMessage.new("abc", { "message" => "hello world" } )}
+      let(:message) { MockMessage.new("abc", { "message" => "hello world", "@metadata" => {} } )}
 
       it "extract the event" do
         subject.onNewMessage(ctx, message)
@@ -107,7 +137,7 @@ describe LogStash::Inputs::Beats::MessageListener do
     end
 
     context "when the message is from LSF" do
-      let(:message) { MockMessage.new("abc", { "line" => "hello world" } )}
+      let(:message) { MockMessage.new("abc", { "line" => "hello world", '@metadata' => {} } )}
 
       it "extract the event" do
         subject.onNewMessage(ctx, message)
@@ -118,13 +148,15 @@ describe LogStash::Inputs::Beats::MessageListener do
 
     context "when the message is from any libbeat" do
       #Requires data modeled as Java, not Ruby since the actual code pulls from Java backed (Netty) object
-      data = java.util.HashMap.new
-      data.put("@metadata", java.util.HashMap.new)
-      data.put("metric",  1)
-      data.put("name", "super-stats")
+      let(:data) do
+        d = HashMap.new
+        d.put('@metadata', HashMap.new)
+        d.put('metric',  1)
+        d.put('name', "super-stats")
+        d
+      end
 
       let(:message) { MockMessage.new("abc",  data)}
-      let(:ctx) {OngoingMethodMock.new("getHostAddress", "10.0.0.1")}
 
       it "extract the event" do
         subject.onNewMessage(ctx, message)
@@ -132,10 +164,41 @@ describe LogStash::Inputs::Beats::MessageListener do
         expect(event.get("message")).to be_nil
         expect(event.get("metric")).to eq(1)
         expect(event.get("name")).to eq("super-stats")
-        expect(event.get("[@metadata][ip_address]")).to eq("10.0.0.1")
+        expect(event.get("[@metadata][ip_address]")).to eq(ip_address)
       end
-    end
 
+      context 'when the remote address is nil' do
+        let(:ctx) { OngoingMethodMock.new("remoteAddress", nil)}
+
+        it 'extracts the event' do
+          subject.onNewMessage(ctx, message)
+          event = queue.pop
+          expect(event.get("message")).to be_nil
+          expect(event.get("metric")).to eq(1)
+          expect(event.get("name")).to eq("super-stats")
+          expect(event.get("[@metadata][ip_address]")).to eq(nil)
+        end
+      end
+
+      context 'when getting the remote address raises' do
+        let(:raising_ctx) { double("context")}
+
+        before  do
+          allow(raising_ctx).to receive(:channel).and_raise("nope")
+          subject.onNewConnection(raising_ctx)
+        end
+
+        it 'extracts the event' do
+          subject.onNewMessage(raising_ctx, message)
+          event = queue.pop
+          expect(event.get("message")).to be_nil
+          expect(event.get("metric")).to eq(1)
+          expect(event.get("name")).to eq("super-stats")
+          expect(event.get("[@metadata][ip_address]")).to eq(nil)
+        end
+      end
+
+    end
   end
 
   context "onException" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 5.0.4
+  version: 5.0.5
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-12 00:00:00.000000000 Z
+date: 2017-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -266,7 +266,7 @@ files:
 - vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/1.1.33.Fork23/netty-tcnative-boringssl-static-1.1.33.Fork23.jar
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
 - vendor/jar-dependencies/org/javassist/javassist/3.20.0-GA/javassist-3.20.0-GA.jar
-- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/5.0.4/logstash-input-beats-5.0.4.jar
+- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/5.0.5/logstash-input-beats-5.0.5.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)