logstash-input-syslog 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: afc32918fa577b058932c4b231b47a8b1f1fe9b568e5126aacf8e9d0224563bb
-  data.tar.gz: 5db1690089b6f82efcad455170defb3f1987d9420197817af225ca2480511ffc
+  metadata.gz: 30915cac1fcc65f8de106471302441df16c4bc5333243faec47223166be5c96b
+  data.tar.gz: ccfa3cf6950b6a4353c6bfcd8ac7073df626b3ed18d0d8cf82e762b80ba9aeb8
 SHA512:
-  metadata.gz: 1e25afad30245aa9bb745861f8fef0d474eb8c0de28c1964e71e939aa02aabe55785fc03312d14771c9608378e045168691e284485facea58dfa48d9a660a581
-  data.tar.gz: d04b48e1192204d932cdb011dfd79ae310370dacb9fb5120b0c2421cbeb2a7d82119eb8719fc31810646b487699d32b453350654c00ca4aa3b7f3db3abf4e333
+  metadata.gz: d515b8ba355a0ad289dc276ca2620a8ebef47304b907abbc003eb975ad5f21f82a63ece8125a30b24ec36dd9077a38675cf34e049d3972d6da0ed4d7dc74565c
+  data.tar.gz: '038276f3c5172c192782ba4d60c98a1a407754fc9290266c94e420d98c0fef06de2fe1afbfc9509c5fe269aa0cb9ffb33cfc39f08209abaaf3d3b94e1c42ae77'

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 3.4.0
+  - Allow the syslog field to be a configurable option.  This is useful for when codecs change
+    the field containing the syslog data (e.g. the CEF codec).
+
 ## 3.3.0
   - Make the grok pattern a configurable option
 

data/docs/index.asciidoc

@@ -53,6 +53,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-proxy_protocol>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-severity_labels>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-syslog_field>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-timezone>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-use_labels>> |<<boolean,boolean>>|No
 |=======================================================================
@@ -136,6 +137,29 @@ http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt
 
 Labels for severity levels. These are defined in RFC3164.
 
+[id="plugins-{type}s-{plugin}-syslog_field"]
+===== `syslog_field`
+
+  * Value type is <<string,string>>
+  * Default value is `"message"`
+
+Codecs process the data before the rest of the data is parsed. Some codecs,
+like CEF, put the syslog data into another field after pre-processing the
+data.  Use this option in conjunction with the `grok_pattern` configuration
+to allow the syslog input plugin to fully parse the syslog data in this case.
+
+[source,sh]
+-------
+input {
+  syslog {
+    port => 12345
+    codec => cef
+    syslog_field => "syslog"
+    grok_pattern => "<%{POSINT:priority}>%{SYSLOGTIMESTAMP:timestamp} CUSTOM GROK HERE"
+  }
+}
+-------
+
 [id="plugins-{type}s-{plugin}-timezone"]
 ===== `timezone`
 

data/lib/logstash/inputs/syslog.rb

@@ -36,6 +36,10 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   # ports) may require root to use.
   config :port, :validate => :number, :default => 514
 
+  # Use custom post-codec processing field (e.g. syslog, after cef codec
+  # processing) instead of the default `message` field
+  config :syslog_field, :validate => :string, :default => "message"
+
   # Set custom grok pattern to parse the syslog, in case the format differs
   # from the defined standard.  This is common in security and other appliances
   config :grok_pattern, :validate => :string, :default => "<%{POSINT:priority}>%{SYSLOGLINE}"
@@ -82,8 +86,8 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     @metric_errors = metric.namespace(:errors)
     require "thread_safe"
     @grok_filter = LogStash::Filters::Grok.new(
-      "overwrite" => "message",
-      "match" => { "message" => @grok_pattern },
+      "overwrite" => @syslog_field,
+      "match" => { @syslog_field => @grok_pattern },
       "tag_on_failure" => ["_grokparsefailure_sysloginput"],
     )
 

data/logstash-input-syslog.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-syslog'
-  s.version         = '3.3.0'
+  s.version         = '3.4.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads syslog messages as events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -31,5 +31,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-filter-date'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-cef'
 end
 

data/spec/inputs/syslog_spec.rb

@@ -22,6 +22,7 @@ module LogStash::Environment
 end
 
 require "logstash/inputs/syslog"
+require "logstash/codecs/cef"
 require "logstash/event"
 require "stud/try"
 require "socket"
@@ -248,4 +249,43 @@ describe LogStash::Inputs::Syslog do
       insist { event.get("timestamp") } == timestamp
     end
   end
+
+  it "should properly handle the cef codec with a custom grok_pattern" do
+    port = 5511
+    event_count = 1
+    custom_grok = "<%{POSINT:priority}>%{TIMESTAMP_ISO8601:timestamp} atypical"
+    message_field = "Description Omitted"
+    timestamp = "2018-02-07T12:40:00.000Z"
+    custom_line = "<134>#{timestamp} atypical CEF:0|Company Name|Application Name|Application Version Number|632|Syslog Configuration Updated|3|src=192.168.0.1 suser=user@example.com target=TARGET msg=#{message_field} KeyValueOne=kv1 KeyValueTwo=12345 "
+
+    conf = <<-CONFIG
+      input {
+        syslog {
+          port => #{port}
+          syslog_field => "syslog"
+          grok_pattern => "#{custom_grok}"
+          codec => cef
+        }
+      }
+    CONFIG
+
+    events = input(conf) do |pipeline, queue|
+      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+      event_count.times do |i|
+        socket.puts(custom_line)
+      end
+      socket.close
+
+      event_count.times.collect { queue.pop }
+    end
+
+    insist { events.length } == event_count
+    events.each do |event|
+      insist { event.get("priority")  } == 134
+      insist { event.get("severity")  } == 6
+      insist { event.get("facility")  } == 16
+      insist { event.get("message")   } == message_field
+      insist { event.get("timestamp") } == timestamp
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-syslog
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-06 00:00:00.000000000 Z
+date: 2018-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -134,6 +134,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-cef
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program