logstash-input-syslog 3.2.4 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a308c5702d6887cf5bf7842fdd4b3cd37d8b3591bb9abf84914473b5711ffffd
-  data.tar.gz: bda506472180eed7313a59b0927b589658ac3344484cfef7b245bbb463a92611
+  metadata.gz: afc32918fa577b058932c4b231b47a8b1f1fe9b568e5126aacf8e9d0224563bb
+  data.tar.gz: 5db1690089b6f82efcad455170defb3f1987d9420197817af225ca2480511ffc
 SHA512:
-  metadata.gz: 86baef4a8e94743bb2523eede569072947d26cddede101162c72b1ee17ed74be9aef62232f4a4d2f0a7b2b4adbe3134051abf6ba91c5b17d2ea7125d255da0be
-  data.tar.gz: 5073d8e96af959e9d71cda4d77e11a2224249733c95386cf818277ed35e6a19b56c43bf33f5efee7039bcab64db771204d5da5b83c9d887d7d589273f0a7b642
+  metadata.gz: 1e25afad30245aa9bb745861f8fef0d474eb8c0de28c1964e71e939aa02aabe55785fc03312d14771c9608378e045168691e284485facea58dfa48d9a660a581
+  data.tar.gz: d04b48e1192204d932cdb011dfd79ae310370dacb9fb5120b0c2421cbeb2a7d82119eb8719fc31810646b487699d32b453350654c00ca4aa3b7f3db3abf4e333

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.3.0
+  - Make the grok pattern a configurable option
+
 ## 3.2.4
   - Fix issue where stopping a pipeline (e.g., while reloading configuration) with active inbound syslog connections could cause Logstash to crash
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/NOTICE.TXT

@@ -1,5 +1,5 @@
 Elasticsearch
-Copyright 2012-2015 Elasticsearch
+Copyright 2012-2018 Elasticsearch
 
 This product includes software developed by The Apache Software
 Foundation (http://www.apache.org/).

data/docs/index.asciidoc

@@ -27,10 +27,11 @@ It is also a good choice if you want to receive logs from
 appliances and network devices where you cannot run your own
 log collector.
 
-Of course, 'syslog' is a very muddy term. This input only supports `RFC3164`
-syslog with some small modifications. The date format is allowed to be
-`RFC3164` style or `ISO8601`. Otherwise the rest of `RFC3164` must be obeyed.
-If you do not use `RFC3164`, do not use this input.
+Of course, 'syslog' is a very muddy term. By default, this input only
+supports `RFC3164` syslog with some small modifications. However, some
+non-standard syslog formats can be read and parsed if a functional
+`grok_pattern` is provided. The date format is still only allowed to be
+`RFC3164` style or `ISO8601`.
 
 For more information see the http://www.ietf.org/rfc/rfc3164.txt[RFC3164 page].
 
@@ -46,6 +47,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-facility_labels>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-grok_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-locale>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
@@ -61,15 +63,33 @@ input plugins.
 &nbsp;
 
 [id="plugins-{type}s-{plugin}-facility_labels"]
-===== `facility_labels` 
+===== `facility_labels`
 
   * Value type is <<array,array>>
   * Default value is `["kernel", "user-level", "mail", "system", "security/authorization", "syslogd", "line printer", "network news", "UUCP", "clock", "security/authorization", "FTP", "NTP", "log audit", "log alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7"]`
 
 Labels for facility levels. These are defined in RFC3164.
 
+[id="plugins-{type}s-{plugin}-grok_pattern"]
+===== `grok_pattern`
+
+  * Value type is <<string,string>>
+  * Default value is `"<%{POSINT:priority}>%{SYSLOGLINE}"`
+
+The default value should read and properly parse syslog lines which are
+fully compliant with http://www.ietf.org/rfc/rfc3164.txt[RFC3164].
+
+You can override this value to parse non-standard lines with a valid grok
+pattern which will parse the received lines.  If the line is unable to
+be parsed, the `_grokparsefailure_sysloginput` tag will be added.
+
+The grok pattern must provide a `timestamp` field. If the `timestamp`
+field is omitted, or is unable to be parsed as `RFC3164` style or
+`ISO8601`, a `_dateparsefailure` tag will be added.
+
+
 [id="plugins-{type}s-{plugin}-host"]
-===== `host` 
+===== `host`
 
   * Value type is <<string,string>>
   * Default value is `"0.0.0.0"`
@@ -77,7 +97,7 @@ Labels for facility levels. These are defined in RFC3164.
 The address to listen on.
 
 [id="plugins-{type}s-{plugin}-locale"]
-===== `locale` 
+===== `locale`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -91,7 +111,7 @@ weekday names (pattern with EEE).
 
 
 [id="plugins-{type}s-{plugin}-port"]
-===== `port` 
+===== `port`
 
   * Value type is <<number,number>>
   * Default value is `514`
@@ -100,7 +120,7 @@ The port to listen on. Remember that ports less than 1024 (privileged
 ports) may require root to use.
 
 [id="plugins-{type}s-{plugin}-proxy_protocol"]
-===== `proxy_protocol` 
+===== `proxy_protocol`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -109,7 +129,7 @@ Proxy protocol support, only v1 is supported at this time
 http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt
 
 [id="plugins-{type}s-{plugin}-severity_labels"]
-===== `severity_labels` 
+===== `severity_labels`
 
   * Value type is <<array,array>>
   * Default value is `["Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug"]`
@@ -117,7 +137,7 @@ http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt
 Labels for severity levels. These are defined in RFC3164.
 
 [id="plugins-{type}s-{plugin}-timezone"]
-===== `timezone` 
+===== `timezone`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -131,7 +151,7 @@ Canonical ID is good as it takes care of daylight saving time for you
 For example, `America/Los_Angeles` or `Europe/France` are valid IDs.
 
 [id="plugins-{type}s-{plugin}-use_labels"]
-===== `use_labels` 
+===== `use_labels`
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
@@ -141,4 +161,4 @@ Use label parsing for severity and facility levels.
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/syslog.rb

@@ -36,6 +36,10 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   # ports) may require root to use.
   config :port, :validate => :number, :default => 514
 
+  # Set custom grok pattern to parse the syslog, in case the format differs
+  # from the defined standard.  This is common in security and other appliances
+  config :grok_pattern, :validate => :string, :default => "<%{POSINT:priority}>%{SYSLOGLINE}"
+
   # Proxy protocol support, only v1 is supported at this time
   # http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt
   config :proxy_protocol, :validate => :boolean, :default => false
@@ -79,12 +83,12 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     require "thread_safe"
     @grok_filter = LogStash::Filters::Grok.new(
       "overwrite" => "message",
-      "match" => { "message" => "<%{POSINT:priority}>%{SYSLOGLINE}" },
+      "match" => { "message" => @grok_pattern },
       "tag_on_failure" => ["_grokparsefailure_sysloginput"],
     )
 
     @date_filter = LogStash::Filters::Date.new(
-      "match" => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601"],
+      "match" => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss", "ISO8601"],
       "locale" => @locale,
       "timezone" => @timezone,
     )

data/logstash-input-syslog.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-syslog'
-  s.version         = '3.2.4'
+  s.version         = '3.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads syslog messages as events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/syslog_spec.rb

@@ -210,4 +210,42 @@ describe LogStash::Inputs::Syslog do
   it_behaves_like 'an interruptible input plugin' do
     let(:config) { { "port" => 5511 } }
   end
+
+  it "should properly handle a custom grok_pattern" do
+    port = 5511
+    event_count = 1
+    custom_grok = "<%{POSINT:priority}>%{SYSLOGTIMESTAMP:timestamp} atypical %{GREEDYDATA:message}"
+    message_field = "This part constitutes the message field"
+    timestamp = "Oct 26 15:19:25"
+    custom_line = "<164>#{timestamp} atypical #{message_field}"
+
+    conf = <<-CONFIG
+      input {
+        syslog {
+          type => "blah"
+          port => #{port}
+          grok_pattern => "#{custom_grok}"
+        }
+      }
+    CONFIG
+
+    events = input(conf) do |pipeline, queue|
+      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+      event_count.times do |i|
+        socket.puts(custom_line)
+      end
+      socket.close
+
+      event_count.times.collect { queue.pop }
+    end
+
+    insist { events.length } == event_count
+    events.each do |event|
+      insist { event.get("priority")  } == 164
+      insist { event.get("severity")  } == 4
+      insist { event.get("facility")  } == 20
+      insist { event.get("message")   } == "#{message_field}\n"
+      insist { event.get("timestamp") } == timestamp
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-syslog
 version: !ruby/object:Gem::Version
-  version: 3.2.4
+  version: 3.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-06 00:00:00.000000000 Z
+date: 2018-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement