logstash-input-syslog 3.4.3 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc9057a2c876b1bdb746a10c07fcc32b66d265aeb5658e1116fb0435bd29ea76
-  data.tar.gz: f33e03b2bb5cdad71d2152c6d563db6c2a1d7175e7cb24d6f5b872d707dbfc42
+  metadata.gz: ef06714d6d1b2383646d6ec53550171592e1dbcd97e3e7b1e847edcceba41d4a
+  data.tar.gz: acd1005f0b2db5ad66ff95c7d74027ead6841d97b4805c05ed19225fa5af3cf5
 SHA512:
-  metadata.gz: 877bd6b44875b1b4318e08073d53caef55c90dbe636ab49a0e7c4e2af5d3c6f8a12527250e0988442aa12e884eee8902db89803cd0a517aa0506bb1127de4ea1
-  data.tar.gz: 477cd68ad7aedb6125205b4c88d0ec2bdae3e5e58d82bc91c21317eef919f1047b7c9d4bbc2ccc7e381c447d1af2adda0ed8d69585544fd95d0a65884af91abb
+  metadata.gz: ea924072e2e8904864649a6be3706523dca20a01c1c7aef441fea1c643f93e78dcb12d869ffb1f4d16ee2ebbdcdf4a396f45d6b3fc77909ad247941a849f9e6d
+  data.tar.gz: d3d658703fa56537a9818f65b92ddba37e8f56e760beda02698b1782c5f89eb9dc6c279c18a0611ecf653f3c9b07894549edf5b750855fc0f42500862f7d6f4a

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 3.6.0
+  - Add support for ECS v8 as alias to v1 implementation [#68](https://github.com/logstash-plugins/logstash-input-syslog/pull/68)
+
+## 3.5.0
+  - Feat: ECS compatibility support [#63](https://github.com/logstash-plugins/logstash-input-syslog/pull/63)
+
+## 3.4.5
+  - Added support for listening on IPv6 addresses
+
+## 3.4.4
+  - Refactor: avoid global side-effect + cleanup [#62](https://github.com/logstash-plugins/logstash-input-syslog/pull/62)
+    * avoid setting `BasicSocket.do_not_reverse_lookup` as it has side effects for others 
+
 ## 3.4.3
   - [DOC] Added expanded descriptions and requirements for facility_labels and severity_labels. [#52](https://github.com/logstash-plugins/logstash-input-syslog/pull/52)
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-syslog.svg)](https://travis-ci.org/logstash-plugins/logstash-input-syslog)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-syslog.svg)](https://travis-ci.com/logstash-plugins/logstash-input-syslog)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -47,6 +47,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-facility_labels>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-grok_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
@@ -64,6 +65,20 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: does not use ECS-compatible field names (for example, `priority` for syslog priority)
+    ** `v1`,`v8`: uses fields that are compatible with Elastic Common Schema (for example, `[log][syslog][priority]`)
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the
+{ecs-ref}[Elastic Common Schema (ECS)].
+
 [id="plugins-{type}s-{plugin}-facility_labels"]
 ===== `facility_labels`
 
@@ -84,6 +99,9 @@ the facility_label is not added to the event.
 
   * Value type is <<string,string>>
   * Default value is `"<%{POSINT:priority}>%{SYSLOGLINE}"`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+    ** ECS Compatibility disabled: `"<%{POSINT:priority}>%{SYSLOGLINE}"`
+    ** ECS Compatibility enabled: `"<%{POSINT:[log][syslog][priority]:int}>%{SYSLOGLINE}"`
 
 The default value should read and properly parse syslog lines which are
 fully compliant with http://www.ietf.org/rfc/rfc3164.txt[RFC3164].

data/lib/logstash/inputs/syslog.rb

@@ -6,6 +6,7 @@ require "logstash/filters/grok"
 require "logstash/filters/date"
 require "logstash/inputs/base"
 require "logstash/namespace"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
 require "stud/interval"
 
 # Read syslog messages as events over the network.
@@ -25,6 +26,8 @@ require "stud/interval"
 # Note: This input will start listeners on both TCP and UDP.
 #
 class LogStash::Inputs::Syslog < LogStash::Inputs::Base
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+
   config_name "syslog"
 
   default :codec, "plain"
@@ -42,7 +45,7 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
 
   # Set custom grok pattern to parse the syslog, in case the format differs
   # from the defined standard.  This is common in security and other appliances
-  config :grok_pattern, :validate => :string, :default => "<%{POSINT:priority}>%{SYSLOGLINE}"
+  config :grok_pattern, :validate => :string
 
   # Proxy protocol support, only v1 is supported at this time
   # http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt
@@ -59,8 +62,7 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
 
   # Specify a time zone canonical ID to be used for date parsing.
   # The valid IDs are listed on the [Joda.org available time zones page](http://joda-time.sourceforge.net/timezones.html).
-  # This is useful in case the time zone cannot be extracted from the value,
-  # and is not the platform default.
+  # This is useful in case the time zone cannot be extracted from the value, and is not the platform default.
   # If this is not specified the platform default will be used.
   # Canonical ID is good as it takes care of daylight saving time for you
   # For example, `America/Los_Angeles` or `Europe/France` are valid IDs.
@@ -75,28 +77,67 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   #
   config :locale, :validate => :string
 
-  public
-  def initialize(params)
+  # ECS only option to configure [service][type] value in produced events.
+  #
+  # NOTE: for now, purposefully un-documented as there are other [service] fields we could support,
+  # assuming users would want that (they have specific use-case for LS as syslog server).
+  config :service_type, :validate => :string, :default => 'system'
+
+  def initialize(*params)
     super
-    BasicSocket.do_not_reverse_lookup = true
-  end # def initialize
 
-  public
-  def register
-    @metric_errors = metric.namespace(:errors)
+    @priority_key = ecs_select[disabled:'priority', v1:'[log][syslog][priority]']
+    @facility_key = ecs_select[disabled:'facility', v1:'[log][syslog][facility][code]']
+    @severity_key = ecs_select[disabled:'severity', v1:'[log][syslog][severity][code]']
+
+    @facility_label_key = ecs_select[disabled:'facility_label', v1:'[log][syslog][facility][name]']
+    @severity_label_key = ecs_select[disabled:'severity_label', v1:'[log][syslog][severity][name]']
+
+    @host_key = ecs_select[disabled:'host', v1:'[host][ip]']
+
+    @grok_pattern ||= ecs_select[
+        disabled:"<%{POSINT:#{@priority_key}}>%{SYSLOGLINE}",
+        v1:"<%{POSINT:#{@priority_key}:int}>%{SYSLOGLINE}"
+    ]
 
     @grok_filter = LogStash::Filters::Grok.new(
-      "overwrite" => @syslog_field,
-      "match" => { @syslog_field => @grok_pattern },
-      "tag_on_failure" => ["_grokparsefailure_sysloginput"],
+        "overwrite" => @syslog_field,
+        "match" => { @syslog_field => @grok_pattern },
+        "tag_on_failure" => ["_grokparsefailure_sysloginput"],
+        "ecs_compatibility" => ecs_compatibility # use ecs-compliant patterns
     )
 
+    @grok_filter_exec = ecs_select[
+        disabled: -> (event) { @grok_filter.filter(event) },
+        v1: -> (event) {
+          event.set('[event][original]', event.get(@syslog_field))
+          @grok_filter.filter(event)
+          set_service_fields(event)
+        }
+    ]
+
     @date_filter = LogStash::Filters::Date.new(
-      "match" => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss", "ISO8601"],
-      "locale" => @locale,
-      "timezone" => @timezone,
+        "match" => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss", "MMM d HH:mm:ss", "ISO8601"],
+        "locale" => @locale,
+        "timezone" => @timezone,
     )
 
+    @date_filter_exec = ecs_select[
+        disabled: -> (event) {
+          # in legacy (non-ecs) mode we used to match (SYSLOGBASE2) timestamp into two fields
+          event.set("timestamp", event.get("timestamp8601")) if event.include?("timestamp8601")
+          @date_filter.filter(event)
+        },
+        v1: -> (event) {
+          @date_filter.filter(event)
+          event.remove('timestamp')
+        }
+    ]
+  end
+
+  def register
+    @metric_errors = metric.namespace(:errors)
+
     @grok_filter.register
     @date_filter.register
 
@@ -104,7 +145,8 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     @tcp = @udp = nil
   end # def register
 
-  public
+  private
+
   def run(output_queue)
     udp_thr = Thread.new(output_queue) do |output_queue|
       server(:udp, output_queue)
@@ -119,8 +161,8 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     udp_thr.join
     tcp_thr.join
   end # def run
+  public :run
 
-  private
   # server call the specified protocol listener and basically restarts on
   # any listener uncatched exception
   #
@@ -137,7 +179,6 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     end
   end
 
-  private
   # udp_listener creates the udp socket and continously read from it.
   # upon exception the socket will be closed and the exception bubbled
   # in the server which will restart the listener
@@ -145,7 +186,8 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     @logger.info("Starting syslog udp listener", :address => "#{@host}:#{@port}")
 
     @udp.close if @udp
-    @udp = UDPSocket.new(Socket::AF_INET)
+    @udp = UDPSocket.new (IPAddr.new(@host).ipv6? rescue nil) ? Socket::AF_INET6 : Socket::AF_INET
+    @udp.do_not_reverse_lookup = true
     @udp.bind(@host, @port)
 
     while !stop?
@@ -157,7 +199,6 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     close_udp
   end # def udp_listener
 
-  private
   # tcp_listener accepts tcp connections and creates a new tcp_receiver thread
   # for each accepted socket.
   # upon exception all tcp sockets will be closed and the exception bubbled
@@ -165,6 +206,7 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   def tcp_listener(output_queue)
     @logger.info("Starting syslog tcp listener", :address => "#{@host}:#{@port}")
     @tcp = TCPServer.new(@host, @port)
+    @tcp.do_not_reverse_lookup = true
 
     while !stop?
       socket = @tcp.accept
@@ -182,11 +224,14 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   # tcp_receiver is executed in a thread, any uncatched exception will be bubbled up to the
   # tcp server thread and all tcp connections will be closed and the listener restarted.
   def tcp_receiver(output_queue, socket)
-    ip, port = socket.peeraddr[3], socket.peeraddr[1]
-    first_read = true
+    peer_addr = socket.peeraddr
+    ip, port = peer_addr[3], peer_addr[1]
+
     @logger.info("new connection", :client => "#{ip}:#{port}")
     LogStash::Util::set_thread_name("input|syslog|tcp|#{ip}:#{port}}")
 
+    first_read = true
+
     socket.each do |line|
       metric.increment(:messages_received)
       if @proxy_protocol && first_read
@@ -194,10 +239,10 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
         pp_info = line.split(/\s/)
         # PROXY proto clientip proxyip clientport proxyport
         if pp_info[0] != "PROXY"
-          @logger.error("invalid proxy protocol header label", :hdr => line)
+          @logger.error("invalid proxy protocol header label", header: line)
           raise IOError
         else
-          # would be nice to log the proxy host and port data as well, but minimizing changes
+          @logger.debug("proxy protocol detected", header: line)
           ip = pp_info[2]
           port = pp_info[3]
           next
@@ -211,49 +256,45 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   rescue Errno::EBADF
     # swallow connection closed exceptions to avoid bubling up the tcp_listener & server
     logger.info("connection closed", :client => "#{ip}:#{port}")
-  rescue IOError => ioerror
+  rescue IOError => e
     # swallow connection closed exceptions to avoid bubling up the tcp_listener & server
-    raise unless socket.closed? && ioerror.message.include?("closed")
-    logger.info("connection error: #{ioerror.message}")
+    raise(e) unless socket.closed? && e.message.to_s.include?("closed")
+    logger.info("connection error:", :exception => e.class, :message => e.message)
   ensure
     @tcp_sockets.delete(socket)
-    socket.close rescue log_and_squash
+    socket.close rescue log_and_squash(:close_tcp_receiver_socket)
   end
 
-  private
-  def decode(host, output_queue, data)
+  def decode(ip, output_queue, data)
     @codec.decode(data) do |event|
       decorate(event)
-      event.set("host", host)
+      event.set(@host_key, ip)
       syslog_relay(event)
       output_queue << event
       metric.increment(:events)
     end
   rescue => e
     # swallow and log all decoding exceptions, these will never be socket related
-    @logger.error("Error decoding data", :data => data.inspect, :exception => e, :backtrace => e.backtrace)
+    @logger.error("Error decoding data", :data => data.inspect, :exception => e.class, :message => e.message, :backtrace => e.backtrace)
     @metric_errors.increment(:decoding)
   end
 
-  public
+  # @see LogStash::Plugin#close
   def stop
     close_udp
     close_tcp
   end
+  public :stop
 
-  private
   def close_udp
     if @udp
-      @udp.close_read rescue log_and_squash
-      @udp.close_write rescue log_and_squash
+      @udp.close_read rescue log_and_squash(:close_udp_read)
+      @udp.close_write rescue log_and_squash(:close_udp_write)
     end
     @udp = nil
   end
 
-  private
-
-  # Helper for inline rescues, which logs the squashed exception at "TRACE" level
-  # and returns nil.
+  # Helper for inline rescues, which logs the exception at "DEBUG" level and returns nil.
   #
   # Instead of:
   # ~~~ ruby
@@ -261,19 +302,19 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   # ~~~
   # Do:
   # ~~~ ruby
-  #.  foo rescue log_and_squash
+  #.  foo rescue log_and_squash(:foo)
   # ~~~
-  def log_and_squash
-    $! && logger.trace("SQUASHED EXCEPTION: `#{$!.message}` at (`#{caller.first}`)")
+  def log_and_squash(label)
+    $! && logger.debug("#{label} failed:", :exception => $!.class, :message => $!.message)
     nil
   end
 
   def close_tcp
     # If we somehow have this left open, close it.
     @tcp_sockets.each do |socket|
-      socket.close rescue log_and_squash
+      socket.close rescue log_and_squash(:close_tcp_socket)
     end
-    @tcp.close if @tcp rescue log_and_squash
+    @tcp.close if @tcp rescue log_and_squash(:close_tcp)
     @tcp = nil
   end
 
@@ -282,46 +323,54 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   # If the message cannot be recognized (see @grok_filter), we'll
   # treat it like the whole event["message"] is correct and try to fill
   # the missing pieces (host, priority, etc)
-  public
   def syslog_relay(event)
-    @grok_filter.filter(event)
+    @grok_filter_exec.(event)
 
     if event.get("tags").nil? || !event.get("tags").include?(@grok_filter.tag_on_failure)
       # Per RFC3164, priority = (facility * 8) + severity
       #                       = (facility << 3) & (severity)
-      priority = event.get("priority").to_i rescue 13
-      severity = priority & 7   # 7 is 111 (3 bits)
-      facility = priority >> 3
-      event.set("priority", priority)
-      event.set("severity", severity)
-      event.set("facility", facility)
-
-      event.set("timestamp", event.get("timestamp8601")) if event.include?("timestamp8601")
-      @date_filter.filter(event)
+      priority = event.get(@priority_key).to_i rescue 13
+      set_priority event, priority
+
+      @date_filter_exec.(event)
+
     else
-      @logger.debug? && @logger.debug("NOT SYSLOG", :message => event.get("message"))
+      @logger.debug? && @logger.debug("un-matched syslog message", :message => event.get("message"))
 
       # RFC3164 says unknown messages get pri=13
-      priority = 13
-      event.set("priority", 13)
-      event.set("severity", 5)   # 13 & 7 == 5
-      event.set("facility", 1)   # 13 >> 3 == 1
+      set_priority event, 13
       metric.increment(:unknown_messages)
     end
 
-    # Apply severity and facility metadata if
-    # use_labels => true
-    if @use_labels
-      facility_number = event.get("facility")
-      severity_number = event.get("severity")
+    # Apply severity and facility metadata if use_labels => true
+    set_labels(event) if @use_labels
+  end # def syslog_relay
+  public :syslog_relay
+
+  def set_priority(event, priority)
+    severity = priority & 7 # 7 is 111 (3 bits)
+    facility = priority >> 3
+    event.set(@priority_key, priority)
+    event.set(@severity_key, severity)
+    event.set(@facility_key, facility)
+  end
 
-      if @facility_labels[facility_number]
-        event.set("facility_label", @facility_labels[facility_number])
-      end
+  def set_labels(event)
+    facility_number = event.get(@facility_key)
+    severity_number = event.get(@severity_key)
 
-      if @severity_labels[severity_number]
-        event.set("severity_label", @severity_labels[severity_number])
-      end
+    facility_label = @facility_labels[facility_number]
+    event.set(@facility_label_key, facility_label) if facility_label
+
+    severity_label = @severity_labels[severity_number]
+    event.set(@severity_label_key, severity_label) if severity_label
+  end
+
+  def set_service_fields(event)
+    service_type = @service_type
+    if service_type && !service_type.empty?
+      event.set('[service][type]', service_type) unless event.include?('[service][type]')
     end
-  end # def syslog_relay
+  end
+
 end # class LogStash::Inputs::Syslog

data/logstash-input-syslog.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-syslog'
-  s.version         = '3.4.3'
+  s.version         = '3.6.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads syslog messages as events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,16 +21,16 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.2'
 
   s.add_runtime_dependency 'concurrent-ruby'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
 
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'logstash-filter-grok'
+  s.add_runtime_dependency 'logstash-filter-grok', '>= 4.4.1'
   s.add_runtime_dependency 'logstash-filter-date'
 
-  s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'insist'
+  s.add_development_dependency 'logstash-devutils', '~> 2.3'
   s.add_development_dependency 'logstash-codec-cef'
 end
 

data/spec/inputs/syslog_spec.rb

@@ -1,8 +1,9 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
-require "insist"
 require "logstash/devutils/rspec/shared_examples"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+
 # running the grok code outside a logstash package means
 # LOGSTASH_HOME will not be defined, so let's set it here
 # before requiring the grok filter
@@ -33,7 +34,7 @@ describe LogStash::Inputs::Syslog do
   SYSLOG_LINE = "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group \"acl_drac\" [0x0, 0x0]"
 
   it "should properly handle priority, severity and facilities" do
-    skip 'elastic/logstash#11196 known LS 7.5 issue' if ENV['ELASTIC_STACK_VERSION'] && JRUBY_VERSION.eql?('9.2.8.0')
+    skip_if_stack_known_issue
     port = 5511
     event_count = 10
     conf = <<-CONFIG
@@ -55,16 +56,16 @@ describe LogStash::Inputs::Syslog do
       event_count.times.collect { queue.pop }
     end
 
-    insist { events.length } == event_count
+    expect( events.length ).to eql event_count
     events.each do |event|
-      insist { event.get("priority") } == 164
-      insist { event.get("severity") } == 4
-      insist { event.get("facility") } == 20
+      expect( event.get("priority") ).to eql 164
+      expect( event.get("severity") ).to eql 4
+      expect( event.get("facility") ).to eql 20
     end
   end
 
   it "should properly PROXY protocol v1" do
-    skip 'elastic/logstash#11196 known LS 7.5 issue' if ENV['ELASTIC_STACK_VERSION'] && JRUBY_VERSION.eql?('9.2.8.0')
+    skip_if_stack_known_issue
     port = 5511
     event_count = 10
     conf = <<-CONFIG
@@ -89,110 +90,200 @@ describe LogStash::Inputs::Syslog do
       event_count.times.collect { queue.pop }
     end
 
-    insist { events.length } == event_count
+    expect( events.length ).to eql event_count
     events.each do |event|
-      insist { event.get("priority") } == 164
-      insist { event.get("severity") } == 4
-      insist { event.get("facility") } == 20
-      insist { event.get("host") } == "1.2.3.4"
+      expect( event.get("priority") ).to eql 164
+      expect( event.get("severity") ).to eql 4
+      expect( event.get("facility") ).to eql 20
+      expect( event.get("host") ).to eql "1.2.3.4"
     end
   end
 
-  it "should add unique tag when grok parsing fails with live syslog input" do
-    skip 'elastic/logstash#11196 known LS 7.5 issue' if ENV['ELASTIC_STACK_VERSION'] && JRUBY_VERSION.eql?('9.2.8.0')
-    port = 5511
-    event_count = 10
-    conf = <<-CONFIG
-      input {
-        syslog {
-          type => "blah"
-          port => #{port}
-        }
-      }
-    CONFIG
+  context 'tag', :ecs_compatibility_support do
+    ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do
 
-    events = input(conf) do |pipeline, queue|
-      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
-      event_count.times do |i|
-        socket.puts("message which causes the a grok parse failure")
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
       end
-      socket.close
 
-      event_count.times.collect { queue.pop }
-    end
+      it "should add unique tag when grok parsing fails with live syslog input" do
+        skip_if_stack_known_issue
+        port = 5511
+        event_count = 10
+        conf = <<-CONFIG
+          input {
+            syslog {
+              type => "blah"
+              port => #{port}
+            }
+          }
+        CONFIG
+
+        events = input(conf) do |pipeline, queue|
+          socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+          event_count.times do |i|
+            socket.puts("message which causes the a grok parse failure")
+          end
+          socket.close
+
+          event_count.times.collect { queue.pop }
+        end
+
+        expect( events.length ).to eql event_count
+        event_count.times do |i|
+          expect( events[i].get("tags") ).to eql ["_grokparsefailure_sysloginput"]
+        end
+      end
 
-    insist { events.length } == event_count
-    event_count.times do |i|
-      insist { events[i].get("tags") } == ["_grokparsefailure_sysloginput"]
     end
   end
 
-  it "should properly handle locale and timezone" do
-    port = 5511
-    event_count = 10
+  context 'timestamp', :ecs_compatibility_support do
+    ecs_compatibility_matrix(:disabled, :v1) do
 
-    conf = <<-CONFIG
-      input {
-        syslog {
-          type => "blah"
-          port => #{port}
-          locale => "en"
-          timezone => "UTC"
-        }
-      }
-    CONFIG
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
 
-    events = input(conf) do |pipeline, queue|
-      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
-      event_count.times do |i|
-        socket.puts(SYSLOG_LINE)
+      it "should properly handle locale and timezone" do
+        port = 5511
+        event_count = 10
+
+        conf = <<-CONFIG
+          input {
+            syslog {
+              type => "blah"
+              port => #{port}
+              locale => "en"
+              timezone => "UTC"
+            }
+          }
+        CONFIG
+
+        events = input(conf) do |pipeline, queue|
+          socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+          event_count.times do |i|
+            socket.puts(SYSLOG_LINE)
+          end
+          socket.close
+
+          event_count.times.collect { queue.pop }
+        end
+
+        expect( events.length ).to eql event_count
+        events.each do |event|
+          expect( event.get("@timestamp") ).to be_a_logstash_timestamp_equivalent_to("#{Time.now.year}-10-26T15:19:25Z")
+        end
       end
-      socket.close
 
-      event_count.times.collect { queue.pop }
-    end
+      it "should properly handle no locale and no timezone" do
+        port = 5511
+
+        conf = <<-CONFIG
+          input {
+            syslog {
+              type => "blah"
+              port => #{port}
+            }
+          }
+        CONFIG
+
+        event = input(conf) do |pipeline, queue|
+          socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+          socket.puts(SYSLOG_LINE)
+          socket.close
+
+          queue.pop
+        end
+
+        # chances platform timezone is not UTC, so parse without offset to create expectation
+        equivalent_time = Time.parse("#{Time.now.year}-10-26T15:19:25")
+        expect( event.get("@timestamp") ).to be_a_logstash_timestamp_equivalent_to(equivalent_time)
+      end
+
+      it "should support non UTC timezone" do
+        input = LogStash::Inputs::Syslog.new({"timezone" => "-05:00"})
+        input.register
+
+        # event which is not syslog should have a new tag
+
+        syslog_event = LogStash::Event.new({ "message" => "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434" })
+        input.syslog_relay(syslog_event)
+
+        expect( syslog_event.get("@timestamp") ).to be_a_logstash_timestamp_equivalent_to("#{Time.now.year}-10-26T20:19:25Z")
+
+        input.close
+      end
 
-    insist { events.length } == event_count
-    events.each do |event|
-      insist { event.get("@timestamp").to_iso8601 } == "#{Time.now.year}-10-26T15:19:25.000Z"
     end
   end
 
-  it "should properly handle no locale and no timezone" do
-    port = 5511
+  context 'ECS behavior', :ecs_compatibility_support do
 
-    conf = <<-CONFIG
-      input {
-        syslog {
-          type => "blah"
-          port => #{port}
-        }
-      }
-    CONFIG
+    ecs_compatibility_matrix(:v1) do
 
-    event = input(conf) do |pipeline, queue|
-      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
-      socket.puts(SYSLOG_LINE)
-      socket.close
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
 
-      queue.pop
-    end
+      let(:event) do
+        LogStash::Event.new("message" => "<164>Oct 26 15:19:25 1.2.3.4 a sample message")
+      end
 
-    # chances platform timezone is not UTC so ignore the hours
-    insist { event.get("@timestamp").to_iso8601 } =~ /#{Time.now.year}-10-26T\d\d:19:25.000Z/
-  end
+      subject { LogStash::Inputs::Syslog.new }
 
-  it "should support non UTC timezone" do
-    input = LogStash::Inputs::Syslog.new({"timezone" => "-05:00"})
-    input.register
+      before { subject.register }
+      after { subject.close }
 
-    # event which is not syslog should have a new tag
+      it "should not have a timestamp field" do
+        subject.syslog_relay(event)
 
-    syslog_event = LogStash::Event.new({ "message" => "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434" })
-    input.syslog_relay(syslog_event)
-    insist { syslog_event.get("@timestamp").to_iso8601 } == "#{Time.now.year}-10-26T20:19:25.000Z"
+        expect( event.to_hash.keys ).to_not include 'timestamp'
+      end
 
-    input.close
+      it "overwrites message" do
+        subject.syslog_relay(event)
+
+        expect( event.get('message') ).to eql 'a sample message'
+      end
+
+      it "keep original log message" do
+        subject.syslog_relay(event)
+
+        expect( event.get('[event][original]') ).to eql '<164>Oct 26 15:19:25 1.2.3.4 a sample message'
+      end
+
+      it "sets syslog priority and severity" do
+        subject.syslog_relay(event)
+
+        expect( event.get('log') ).to include 'syslog' => hash_including('priority' => 164)
+        expect( event.get('log') ).to include 'syslog' => hash_including('severity' => { 'code' => 4, 'name' => 'Warning' })
+      end
+
+      it "sets service type" do
+        subject.syslog_relay(event)
+
+        expect( event.get('service') ).to include 'type' => 'system'
+      end
+
+      let(:queue) { Queue.new }
+
+      let(:socket) do
+        server = double('tcp-server')
+        allow( server ).to receive(:each).and_yield "<133>Mar 11 08:44:43 precision kernel: [765135.424096] mce: CPU6: Package temperature/speed normal\n"
+        allow( server ).to receive(:close)
+        server
+      end
+
+      it "sets host IP" do
+        expect( socket ).to receive(:peeraddr).and_return(["AF_INET", 514, "192.168.0.10", "192.168.0.10"])
+        subject.send :tcp_receiver, queue, socket
+
+        expect( queue.size ).to eql 1
+        event = queue.pop
+        expect( event.get('host') ).to eql 'hostname' => 'precision', 'ip' => '192.168.0.10'
+      end
+    end
   end
 
   it "should add unique tag when grok parsing fails" do
@@ -202,13 +293,13 @@ describe LogStash::Inputs::Syslog do
     # event which is not syslog should have a new tag
     event = LogStash::Event.new({ "message" => "hello world, this is not syslog RFC3164" })
     input.syslog_relay(event)
-    insist { event.get("tags") } ==  ["_grokparsefailure_sysloginput"]
+    expect( event.get("tags") ).to eql  ["_grokparsefailure_sysloginput"]
 
     syslog_event = LogStash::Event.new({ "message" => "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434" })
     input.syslog_relay(syslog_event)
-    insist { syslog_event.get("priority") } ==  164
-    insist { syslog_event.get("severity") } ==  4
-    insist { syslog_event.get("tags") } ==  nil
+    expect( syslog_event.get("priority") ).to eql 164
+    expect( syslog_event.get("severity") ).to eql 4
+    expect( syslog_event.get("tags") ).to be nil
 
     input.close
   end
@@ -245,13 +336,13 @@ describe LogStash::Inputs::Syslog do
       event_count.times.collect { queue.pop }
     end
 
-    insist { events.length } == event_count
+    expect( events.length ).to eql event_count
     events.each do |event|
-      insist { event.get("priority")  } == 164
-      insist { event.get("severity")  } == 4
-      insist { event.get("facility")  } == 20
-      insist { event.get("message")   } == "#{message_field}\n"
-      insist { event.get("timestamp") } == timestamp
+      expect( event.get("priority") ).to eql 164
+      expect( event.get("severity") ).to eql 4
+      expect( event.get("facility") ).to eql 20
+      expect( event.get("message") ).to eql "#{message_field}\n"
+      expect( event.get("timestamp") ).to eql timestamp
     end
   end
 
@@ -284,13 +375,19 @@ describe LogStash::Inputs::Syslog do
       event_count.times.collect { queue.pop }
     end
 
-    insist { events.length } == event_count
+    expect( events.length ).to eql event_count
     events.each do |event|
-      insist { event.get("priority")  } == 134
-      insist { event.get("severity")  } == 6
-      insist { event.get("facility")  } == 16
-      insist { event.get("message")   } == message_field
-      insist { event.get("timestamp") } == timestamp
+      expect( event.get("priority") ).to eql 134
+      expect( event.get("severity") ).to eql 6
+      expect( event.get("facility") ).to eql 16
+      expect( event.get("message") ).to eql message_field
+      expect( event.get("timestamp") ).to eql timestamp
     end
   end
+
+  private
+
+  def skip_if_stack_known_issue
+    skip 'elastic/logstash#11196 known LS 7.5 issue' if ENV['ELASTIC_STACK_VERSION'] && JRUBY_VERSION.eql?('9.2.8.0')
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-syslog
 version: !ruby/object:Gem::Version
-  version: 3.4.3
+  version: 3.6.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2021-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,20 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -83,7 +97,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.4.1
   name: logstash-filter-grok
   prerelease: false
   type: :runtime
@@ -91,7 +105,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.4.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -109,31 +123,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: insist
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -187,8 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Reads syslog messages as events