logstash-input-gelf 3.0.7 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b21ddfaa0568e8cf3d5cbf4634f73b9fd65f46688f5ed825c150c1b0cd3ca606
-  data.tar.gz: 78657fa48ea9e0bf2c89f72100b82590d47e4f7f61e4b9c7f5fb065cee6ef4a9
+  metadata.gz: b380005f437f84a9255e8583c3b3ea1fc6c54e7c7f4ab67aef8f5e674729887a
+  data.tar.gz: fe84ff257d1cffec9cf3c9281e08a9dd3b5d32388eb7c50682ba2ec38db7ce28
 SHA512:
-  metadata.gz: aba07a0185db140cea32f308ab3d2c8f3e137bbb43dc502c1bff459020c2d9f1ff2a245560215eadb90a9abb40a847683bd8b9a4c64a5b1f514753e07f6250d4
-  data.tar.gz: 01b2a5e830938baa44887c65d5feab7ab9ab91254b5eec26e529d73bd8758428b62c816458a2dd858dc73935a0bb76ed6228bab0b2ebe66e6566ad6c93e84176
+  metadata.gz: 2a0aea419cecf15aef6c248b71c3c46098ed974cc534cb32746c416feae7e6c9de03126a399b23ef5540ab5c306aaac9429f629fdd03499387507583a58f9791
+  data.tar.gz: dd6dd7344a6facc9519e9beee80aff02771fea93023af8578dcf3593d97364576e97ff01ca95bb32124aa3a62bf3019615a13b10160c98aed308c0da2e41d6fe

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.0
+  - Add support for listening on TCP socket
+
 ## 3.0.7
   - Update gemspec summary
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -44,7 +44,11 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-use_udp>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-use_tcp>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-port_tcp>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-port_udp>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-remap>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-strip_leading_underscore>> |<<boolean,boolean>>|No
 |=======================================================================
@@ -62,6 +66,22 @@ input plugins.
 
 The IP address or hostname to listen on.
 
+[id="plugins-{type}s-{plugin}-use_udp"]
+===== `use_udp`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Whether to listen for gelf messages sent over udp
+
+[id="plugins-{type}s-{plugin}-use_tcp"]
+===== `use_tcp`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Whether to listen for gelf messages sent over tcp
+
 [id="plugins-{type}s-{plugin}-port"]
 ===== `port` 
 
@@ -70,6 +90,22 @@ The IP address or hostname to listen on.
 
 The port to listen on. Remember that ports less than 1024 (privileged
 ports) may require root to use.
+port_tcp and port_udp can be used to set a specific port for each protocol.
+[id="plugins-{type}s-{plugin}-port_tcp"]
+===== `port_tcp`
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+Tcp port to listen on. Use port instead of this setting unless you need a different port for udp than tcp
+
+[id="plugins-{type}s-{plugin}-port_udp"]
+===== `port_udp`
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+Udp port to listen on. Use port instead of this setting unless you need a different port for udp than tcp
 
 [id="plugins-{type}s-{plugin}-remap"]
 ===== `remap` 
@@ -102,4 +138,4 @@ e.g. `\_foo` becomes `foo`
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/gelf.rb

@@ -6,6 +6,7 @@ require "logstash/timestamp"
 require "stud/interval"
 require "date"
 require "socket"
+require "json"
 
 # This input will read GELF messages as events over the network,
 # making it a good choice if you already use Graylog2 today.
@@ -29,9 +30,12 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
   # The IP address or hostname to listen on.
   config :host, :validate => :string, :default => "0.0.0.0"
 
-  # The port to listen on. Remember that ports less than 1024 (privileged
+  # The ports to listen on. Remember that ports less than 1024 (privileged
   # ports) may require root to use.
+  # port_tcp and port_udp can be used to have a different port for udp than the tcp port.
   config :port, :validate => :number, :default => 12201
+  config :port_tcp, :validate => :number
+  config :port_udp, :validate => :number
 
   # Whether or not to remap the GELF message fields to Logstash event fields or
   # leave them intact.
@@ -59,6 +63,10 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
   PARSE_FAILURE_TAG = "_jsonparsefailure"
   PARSE_FAILURE_LOG_MESSAGE = "JSON parse failure. Falling back to plain-text"
 
+  # Whether or not to use TCP or/and UDP
+  config :use_tcp, :validate => :boolean, :default => false
+  config :use_udp, :validate => :boolean, :default => true
+
   public
   def initialize(params)
     super
@@ -68,13 +76,23 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
   public
   def register
     require 'gelfd'
-  end # def register
+    @port_tcp ||= @port
+    @port_udp ||= @port
+  end
 
   public
   def run(output_queue)
     begin
-      # udp server
-      udp_listener(output_queue)
+      if @use_tcp
+        tcp_thr = Thread.new(output_queue) do |output_queue|
+          tcp_listener(output_queue)
+        end
+      end
+      if @use_udp
+        udp_thr = Thread.new(output_queue) do |output_queue|
+          udp_listener(output_queue)
+        end
+      end
     rescue => e
       unless stop?
         @logger.warn("gelf listener died", :exception => e, :backtrace => e.backtrace)
@@ -82,23 +100,113 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
         retry unless stop?
       end
     end # begin
+    if @use_tcp
+      tcp_thr.join
+    end
+    if @use_udp
+      udp_thr.join
+    end
   end # def run
 
   public
   def stop
-    @udp.close
-  rescue IOError # the plugin is currently shutting down, so its safe to ignore theses errors
+    begin
+      @udp.close if @use_udp
+    rescue IOError => e
+      @logger.warn("Caugh exception while closing udp socket", :exception => e.inspect)
+    end
+    begin
+      @tcp.close if @use_tcp
+    rescue IOError => e
+      @logger.warn("Caugh exception while closing tcp socket", :exception => e.inspect)
+    end
+  end
+
+  private
+  def tcp_listener(output_queue)
+
+    @logger.info("Starting gelf listener (tcp) ...", :address => "#{@host}:#{@port_tcp}")
+
+    if @tcp.nil?
+      @tcp = TCPServer.new(@host, @port_tcp)
+    end
+
+    while !@shutdown_requested
+      Thread.new(@tcp.accept) do |client|
+        @logger.debug? && @logger.debug("Gelf (tcp): Accepting connection from:  #{client.peeraddr[2]}:#{client.peeraddr[1]}")
+
+        begin
+          while !client.nil? && !client.eof?
+
+            begin # Read from socket
+              data_in = client.gets("\u0000")
+            rescue => ex
+              @logger.warn("Gelf (tcp): failed gets from client socket:", :exception => ex, :backtrace => ex.backtrace)
+            end
+
+             if data_in.nil?
+              @logger.warn("Gelf (tcp): socket read succeeded, but data is nil.  Skipping.")
+              next
+            end
+
+            # data received.  Remove trailing \0
+            data_in[-1] == "\u0000" && data_in = data_in[0...-1]
+            begin # Parse JSON
+              jsonObj = JSON.parse(data_in)
+            rescue => ex
+              @logger.warn("Gelf (tcp): failed to parse a message. Skipping: " + data_in, :exception => ex, :backtrace => ex.backtrace)
+              next
+            end
+
+            begin  # Create event
+              event = LogStash::Event.new(jsonObj)
+              event.set(SOURCE_HOST_FIELD, host.force_encoding("UTF-8"))
+              if event.get("timestamp").is_a?(Numeric)
+                event.set("timestamp", LogStash::Timestamp.at(event.get("timestamp")))
+                event.remove("timestamp")
+              end
+              remap_gelf(event) if @remap
+              strip_leading_underscore(event) if @strip_leading_underscore
+              decorate(event)
+              output_queue << event
+            rescue => ex
+              @logger.warn("Gelf (tcp): failed to create event from json object. Skipping: " + jsonObj.to_s, :exception => ex, :backtrace => ex.backtrace)
+            end
+
+          end # while client
+          @logger.debug? && @logger.debug("Gelf (tcp): Closing client connection")
+          client.close
+          client = nil
+        rescue => ex
+          @logger.warn("Gelf (tcp): client socket failed.", :exception => ex, :backtrace => ex.backtrace)
+        ensure
+          if !client.nil?
+            @logger.debug? && @logger.debug("Gelf (tcp): Ensuring client is closed")
+            client.close
+            client = nil
+          end
+        end # begin client
+      end  # Thread.new
+    end # @shutdown_requested
+
   end
 
   private
   def udp_listener(output_queue)
-    @logger.info("Starting gelf listener", :address => "#{@host}:#{@port}")
+    @logger.info("Starting gelf listener (udp) ...", :address => "#{@host}:#{@port_udp}")
 
     @udp = UDPSocket.new(Socket::AF_INET)
-    @udp.bind(@host, @port)
+    @udp.bind(@host, @port_udp)
 
-    while !stop?
-      line, client = @udp.recvfrom(8192)
+    while !@udp.closed?
+      begin
+        line, client = @udp.recvfrom(8192)
+      rescue => e
+        if !stop? # if we're shutting down there's no point in logging anything
+          @logger.error("Caught exception while reading from UDP socket", :exception => e.inspect)
+        end
+        next
+      end
 
       begin
         data = Gelfd::Parser.parse(line)
@@ -129,7 +237,7 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
     event = parse(json_gelf)
     return if event.nil?
 
-    event.set(SOURCE_HOST_FIELD, host)
+    event.set(SOURCE_HOST_FIELD, host.force_encoding("UTF-8"))
 
     if (gelf_timestamp = event.get(TIMESTAMP_GELF_FIELD)).is_a?(Numeric)
       event.timestamp = self.coerce_timestamp(gelf_timestamp)

data/logstash-input-gelf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-gelf'
-  s.version         = '3.0.7'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads GELF-format messages from Graylog2 as events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/gelf_spec.rb

@@ -16,36 +16,34 @@ describe LogStash::Inputs::Gelf do
     before { producer.run }
     after { producer.stop }
 
-
     it_behaves_like "an interruptible input plugin"
   end
 
-  it "reads chunked gelf messages " do
-    port = 12209
-    host = "127.0.0.1"
-    chunksize = 1420
-    gelfclient = GELF::Notifier.new(host, port, chunksize)
+  describe "chunked gelf messages" do
+    let(:port) { 12209 }
+    let(:host) { "127.0.0.1" }
+    let(:chunksize) { 1420 }
+    let(:gelfclient) { GELF::Notifier.new(host, port, chunksize) }
+    let(:large_random) { 2000.times.map{32 + rand(126 - 32)}.join("") }
 
-    conf = <<-CONFIG
-      input {
-        gelf {
-          port => "#{port}"
-          host => "#{host}"
-        }
-      }
-    CONFIG
+    let(:config) { { "port" => port, "host" => host } }
+    let(:queue) { Queue.new }
 
-    large_random = 2000.times.map{32 + rand(126 - 32)}.join("")
+    subject { described_class.new(config) }
 
-    messages = [
+    let(:messages) { [
       "hello",
       "world",
       large_random,
       "we survived gelf!"
-    ]
+    ] }
 
-    events = input(conf) do |pipeline, queue|
-      # send a first message until plugin is up and receives it
+    before(:each) do
+      subject.register
+      Thread.new { subject.run(queue) }
+    end
+
+    it "processes them" do
       while queue.size <= 0
         gelfclient.notify!("short_message" => "prime")
         sleep(0.1)
@@ -58,15 +56,15 @@ describe LogStash::Inputs::Gelf do
       end
 
       messages.each do |m|
-  	    gelfclient.notify!("short_message" => m)
+        gelfclient.notify!("short_message" => m)
       end
 
-      messages.map{queue.pop}
-    end
+      events = messages.map{queue.pop}
 
-    events.each_with_index do |e, i|
-      insist { e.get("message") } == messages[i]
-      insist { e.get("host") } == Socket.gethostname
+      events.each_with_index do |e, i|
+        expect(e.get("message")).to eq(messages[i])
+        expect(e.get("host")).to eq(Socket.gethostname)
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-gelf
 version: !ruby/object:Gem::Version
-  version: 3.0.7
+  version: 3.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-14 00:00:00.000000000 Z
+date: 2018-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -155,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Reads GELF-format messages from Graylog2 as events