logstash-input-gelf 3.1.1 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9832ba400d65d31c41c684fa931d708020f132d2595a90c1074bb7d68b23e8b
-  data.tar.gz: 111607b646a20a8a55be158af487524b26a1b81e2a7c862db327c247179f64f7
+  metadata.gz: 998fc138bf6014e94c8a76e60c3e3cfebc544b73b83266d5e9e84ae23b3a5f1d
+  data.tar.gz: 83766fd3878863f1dda826243124f3a3b7358a5b55386456a314226cbfa4508e
 SHA512:
-  metadata.gz: 1e7d4a04aac25dafad63af6b9a296c81ec7e7611202fe9bd94ca10c1e623e1e7a8900877834aa4c8ddc04f4546d83790f5f32c76e98f1847346549ae5e84ba98
-  data.tar.gz: 73fa9493e0a5dc35ea13d4d937c1ea06a5ad4d6c102677f32f0f47118a5bd593a4d5a854daf789fb129e072933128f8ba9f874b4746b553a2a4a8520a183631f
+  metadata.gz: ee88a28f662343cd752f7294a5f37fa1e275e3e5c2a39d946b18d42dc4c8a3d0e77df04cc2b3251b451442789c51afabdd18483f1263f91a25ff1f83249bb2aa
+  data.tar.gz: 38b4765dd2304e76590edb04f37184f450f2bca548a09f45e82b3881f5b1dc2a899cdce08537898fe7dc6b1481ca657b560e21ef44745b9bb98ae6e1c9212ef8

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.2.0
+  - Fixed shutdown handling, robustness in socket closing and restarting, json parsing, code DRYing and cleanups [62](https://github.com/logstash-plugins/logstash-input-gelf/pull/62)
+
 ## 3.1.1
   - Docs: Set the default_codec doc attribute.
 

data/lib/logstash/inputs/gelf.rb

@@ -67,143 +67,136 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
   config :use_tcp, :validate => :boolean, :default => false
   config :use_udp, :validate => :boolean, :default => true
 
-  public
   def initialize(params)
     super
     BasicSocket.do_not_reverse_lookup = true
-  end # def initialize
+  end
 
-  public
   def register
     require 'gelfd'
     @port_tcp ||= @port
     @port_udp ||= @port
   end
 
-  public
   def run(output_queue)
     begin
       if @use_tcp
-        tcp_thr = Thread.new(output_queue) do |output_queue|
-          tcp_listener(output_queue)
-        end
+        @tcp_thr = Thread.new(output_queue) { |output_queue| tcp_listener(output_queue) }
       end
       if @use_udp
-        udp_thr = Thread.new(output_queue) do |output_queue|
-          udp_listener(output_queue)
-        end
+        @udp_thr = Thread.new(output_queue) { |output_queue| udp_listener(output_queue) }
       end
     rescue => e
       unless stop?
         @logger.warn("gelf listener died", :exception => e, :backtrace => e.backtrace)
         Stud.stoppable_sleep(RECONNECT_BACKOFF_SLEEP) { stop? }
-        retry unless stop?
+        if !stop?
+          # before retrying make sure we close all sockets
+          stop
+          wait_server_thread
+          retry
+        end
       end
-    end # begin
-    if @use_tcp
-      tcp_thr.join
-    end
-    if @use_udp
-      udp_thr.join
     end
-  end # def run
 
-  public
+    wait_server_thread
+  end
+
   def stop
     begin
       @udp.close if @use_udp
-    rescue IOError => e
-      @logger.warn("Caugh exception while closing udp socket", :exception => e.inspect)
+      @udp = nil
+    rescue => e
+      @logger.debug("Caught exception while closing udp socket", :exception => e)
     end
     begin
       @tcp.close if @use_tcp
-    rescue IOError => e
-      @logger.warn("Caugh exception while closing tcp socket", :exception => e.inspect)
+      @tcp = nil
+    rescue => e
+      @logger.debug("Caught exception while closing tcp socket", :exception => e)
     end
   end
 
   private
-  def tcp_listener(output_queue)
 
+  def wait_server_thread
+    @tcp_thr.join if @use_tcp
+    @udp_thr.join if @use_udp
+  end
+
+  def tcp_listener(output_queue)
     @logger.info("Starting gelf listener (tcp) ...", :address => "#{@host}:#{@port_tcp}")
 
     if @tcp.nil?
       @tcp = TCPServer.new(@host, @port_tcp)
     end
 
-    while !@shutdown_requested
+    while !stop?
       Thread.new(@tcp.accept) do |client|
-        @logger.debug? && @logger.debug("Gelf (tcp): Accepting connection from:  #{client.peeraddr[2]}:#{client.peeraddr[1]}")
-
         begin
-          while !client.nil? && !client.eof?
+          @logger.debug? && @logger.debug("Gelf (tcp): Accepting connection from:  #{client.peeraddr[2]}:#{client.peeraddr[1]}")
+
+          while !client.nil? && !client.eof? && !stop?
 
             begin # Read from socket
               data_in = client.gets("\u0000")
             rescue => ex
-              @logger.warn("Gelf (tcp): failed gets from client socket:", :exception => ex, :backtrace => ex.backtrace)
+              if !stop?
+                @logger.warn("Gelf (tcp): failed gets from client socket:", :exception => ex, :backtrace => ex.backtrace)
+              end
             end
 
              if data_in.nil?
-              @logger.warn("Gelf (tcp): socket read succeeded, but data is nil.  Skipping.")
+              @logger.debug("Gelf (tcp): socket read succeeded, but data is nil. Skipping.")
               next
-            end
+             end
 
             # data received.  Remove trailing \0
             data_in[-1] == "\u0000" && data_in = data_in[0...-1]
-            begin # Parse JSON
-              jsonObj = JSON.parse(data_in)
-            rescue => ex
-              @logger.warn("Gelf (tcp): failed to parse a message. Skipping: " + data_in, :exception => ex, :backtrace => ex.backtrace)
-              next
-            end
 
-            begin  # Create event
-              event = LogStash::Event.new(jsonObj)
-              event.set(SOURCE_HOST_FIELD, host.force_encoding("UTF-8"))
-              if event.get("timestamp").is_a?(Numeric)
-                event.set("timestamp", LogStash::Timestamp.at(event.get("timestamp")))
-                event.remove("timestamp")
-              end
-              remap_gelf(event) if @remap
-              strip_leading_underscore(event) if @strip_leading_underscore
-              decorate(event)
-              output_queue << event
-            rescue => ex
-              @logger.warn("Gelf (tcp): failed to create event from json object. Skipping: " + jsonObj.to_s, :exception => ex, :backtrace => ex.backtrace)
-            end
+            event = self.class.new_event(data_in, client.peeraddr[3])
+            next if event.nil?
+
+            remap_gelf(event) if @remap
+            strip_leading_underscore(event) if @strip_leading_underscore
+            decorate(event)
+            output_queue << event
+          end
 
-          end # while client
           @logger.debug? && @logger.debug("Gelf (tcp): Closing client connection")
-          client.close
+          client.close rescue nil
           client = nil
         rescue => ex
-          @logger.warn("Gelf (tcp): client socket failed.", :exception => ex, :backtrace => ex.backtrace)
+          if !stop?
+            @logger.warn("Gelf (tcp): client socket failed.", :exception => ex, :backtrace => ex.backtrace)
+          end
         ensure
           if !client.nil?
             @logger.debug? && @logger.debug("Gelf (tcp): Ensuring client is closed")
-            client.close
+            client.close rescue nil
             client = nil
           end
-        end # begin client
-      end  # Thread.new
-    end # @shutdown_requested
-
+        end
+      end
+    end
+  ensure
+    @logger.debug? && @logger.debug("Gelf (tcp): Ensuring tcp server is closed")
+    @tcp.close rescue nil
+    @tcp = nil
   end
 
-  private
   def udp_listener(output_queue)
     @logger.info("Starting gelf listener (udp) ...", :address => "#{@host}:#{@port_udp}")
 
     @udp = UDPSocket.new(Socket::AF_INET)
     @udp.bind(@host, @port_udp)
 
-    while !@udp.closed?
+    while !stop?
       begin
         line, client = @udp.recvfrom(8192)
       rescue => e
-        if !stop? # if we're shutting down there's no point in logging anything
-          @logger.error("Caught exception while reading from UDP socket", :exception => e.inspect)
+        if !stop?
+          @logger.error("Caught exception while reading from UDP socket", :exception => e)
         end
         next
       end
@@ -227,7 +220,11 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
 
       output_queue << event
     end
-  end # def udp_listener
+  ensure
+    @logger.debug? && @logger.debug("Gelf (udp): Ensuring udp socket is closed")
+    @udp.close rescue nil
+    @udp = nil
+  end
 
   # generate a new LogStash::Event from json input and assign host to source_host event field.
   # @param json_gelf [String] GELF json data
@@ -256,32 +253,14 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
     timestamp.is_a?(BigDecimal) ? LogStash::Timestamp.at(timestamp.to_i, timestamp.frac * 1000000) : LogStash::Timestamp.at(timestamp)
   end
 
-  # from_json_parse uses the Event#from_json method to deserialize and directly produce events
-  def self.from_json_parse(json)
-    # from_json will always return an array of item.
-    # in the context of gelf, the payload should be an array of 1
-    LogStash::Event.from_json(json).first
-  rescue LogStash::Json::ParserError => e
-    logger.error(PARSE_FAILURE_LOG_MESSAGE, :error => e, :data => json)
-    LogStash::Event.new(MESSAGE_FIELD => json, TAGS_FIELD => [PARSE_FAILURE_TAG, '_fromjsonparser'])
-  end # def self.from_json_parse
-
-  # legacy_parse uses the LogStash::Json class to deserialize json
-  def self.legacy_parse(json)
+  def self.parse(json)
     o = LogStash::Json.load(json)
     LogStash::Event.new(o)
   rescue LogStash::Json::ParserError => e
-    logger.error(PARSE_FAILURE_LOG_MESSAGE, :error => e, :data => json)
-    LogStash::Event.new(MESSAGE_FIELD => json, TAGS_FIELD => [PARSE_FAILURE_TAG, '_legacyjsonparser'])
-  end # def self.parse
-
-  # keep compatibility with all v2.x distributions. only in 2.3 will the Event#from_json method be introduced
-  # and we need to keep compatibility for all v2 releases.
-  class << self
-    alias_method :parse, LogStash::Event.respond_to?(:from_json) ? :from_json_parse : :legacy_parse
+    logger.error(PARSE_FAILURE_LOG_MESSAGE, :error => e, :data => json.inspect)
+    LogStash::Event.new(MESSAGE_FIELD => json.inspect, TAGS_FIELD => [PARSE_FAILURE_TAG])
   end
 
-  private
   def remap_gelf(event)
     if event.get("full_message") && !event.get("full_message").empty?
       event.set("message", event.get("full_message").dup)
@@ -293,9 +272,8 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
       event.set("message", event.get("short_message").dup)
       event.remove("short_message")
     end
-  end # def remap_gelf
+  end
 
-  private
   def strip_leading_underscore(event)
      # Map all '_foo' fields to simply 'foo'
      event.to_hash.keys.each do |key|
@@ -303,5 +281,5 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
        event.set(key[1..-1], event.get(key))
        event.remove(key)
      end
-  end # deef removing_leading_underscores
-end # class LogStash::Inputs::Gelf
+  end
+end

data/logstash-input-gelf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-gelf'
-  s.version         = '3.1.1'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads GELF-format messages from Graylog2 as events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "gelfd", ["0.2.0"]                 #(Apache 2.0 license)
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency "stud", "~> 0.0.22"
+  s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency "gelf", ["3.0.0"]                  #(MIT license)

data/spec/inputs/gelf_spec.rb

@@ -148,38 +148,15 @@ describe LogStash::Inputs::Gelf do
   context "when an invalid JSON is fed to the listener" do
     subject { LogStash::Inputs::Gelf.new_event(message, "host") }
     let(:message) { "Invalid JSON message" }
+    context "JSON parser output" do
+      it { should be_a(LogStash::Event) }
 
-    if LogStash::Event.respond_to?(:from_json)
-      context "default :from_json parser output" do
-        it { should be_a(LogStash::Event) }
-
-        it "falls back to plain-text" do
-          expect(subject.get("message")).to eq(message)
-        end
-
-        it "tags message with _jsonparsefailure" do
-          expect(subject.get("tags")).to include("_jsonparsefailure")
-        end
-
-        it "tags message with _fromjsonparser" do
-          expect(subject.get("tags")).to include("_fromjsonparser")
-        end
+      it "falls back to plain-text" do
+        expect(subject.get("message")).to eq(message.inspect)
       end
-    else
-      context "legacy JSON parser output" do
-        it { should be_a(LogStash::Event) }
-
-        it "falls back to plain-text" do
-          expect(subject.get("message")).to eq(message)
-        end
-
-        it "tags message with _jsonparsefailure" do
-          expect(subject.get("tags")).to include("_jsonparsefailure")
-        end
 
-        it "tags message with _legacyjsonparser" do
-          expect(subject.get("tags")).to include("_legacyjsonparser")
-        end
+      it "tags message with _jsonparsefailure" do
+        expect(subject.get("tags")).to include("_jsonparsefailure")
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-gelf
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2018-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,17 +61,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
   name: stud
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -155,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Reads GELF-format messages from Graylog2 as events