logstash-codec-cef 4.0.0-java → 4.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c46dcaf722e0c1935b66dfcf7a7d71e3bcc51844
-  data.tar.gz: 160e4b282a46867cbdcada9e3ca1f674ade9343f
+  metadata.gz: f01e9a10dba1242f1db5e742622aa2b919a4e611
+  data.tar.gz: 0381084bd82fe355d71ee4b09da90042cc5d35b4
 SHA512:
-  metadata.gz: b0f028820eb77468b5cb98c2e9a7ebad5205119b9da6fd43b2bd0b193ede44279593cc901a35713c1cf1e7c264e9d2c5bf601cbfe82d79c5a51b4d8086c44a59
-  data.tar.gz: bb86b4781182b84949b29bdcd5659b52f5e3a11c0211a5f2a984051ff182d9cd84c2879ba671bbcf18ab529fc2086399d35cb87e334de6827277f88c65fb2773
+  metadata.gz: 8eb490d24e26857673c34e6deb1b99a8b3be90ed168b015024aa4ad80d9f2565f1cd2b7f99fd761b5a300cf3de616af8d1fb75f2fb5c620930b7136ede4542d4
+  data.tar.gz: 3d64f1093864b14fc3a58f15cc19a3e5ab1942b286d7f35dda70071efd1a992db5a362595bad6fa1345a92bda367e3a43b812dc4af16b4012d2d639bdb534a70

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.0
+ - Add `delimiter` setting. This allows the decoder to be used with inputs like the TCP input where event delimiters are used.
+
 ## 4.0.0
  - Implements the dictionary translation for abbreviated CEF field names from chapter Chapter 2: ArcSight Extension Dictionary    page 3 of 39 [CEF specification](https://protect724.hp.com/docs/DOC-1072).
  - add `_cefparsefailure` tag on failed decode

data/lib/logstash/codecs/cef.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "logstash/util/buftok"
 require "logstash/codecs/base"
 require "json"
 
@@ -39,7 +40,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   # Defined as field of type string to allow sprintf. The value will be validated
   # to be an integer in the range from 0 to 10 (including).
   # All invalid values will be mapped to the default of 6.
-  config :sev, :validate => :string, :default => "6", :deprecated => "This setting is being deprecated, use :severity instead."
+  config :sev, :validate => :string, :deprecated => "This setting is being deprecated, use :severity instead."
 
   # Severity field in CEF header. The new value can include `%{foo}` strings
   # to help you build a new value from other parts of the event.
@@ -55,7 +56,25 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   # Set this flag if you want to have both v1 and v2 fields indexed at the same time. Note that this option will increase 
   # the index size and data stored in outputs like Elasticsearch
   # This option is available to ease transition to new schema
-  config :deprecated_v1_fields, :validate => :boolean, :default => false, :deprecated => "This setting is being deprecated"
+  config :deprecated_v1_fields, :validate => :boolean, :deprecated => "This setting is being deprecated"
+
+  # If your input puts a delimiter between each CEF event, you'll want to set
+  # this to be that delimiter.
+  #
+  # For example, with the TCP input, you probably want to put this:
+  #
+  #     input {
+  #       tcp {
+  #         codec => cef { delimiter => "\r\n" }
+  #         # ... 
+  #       }
+  #     }
+  #
+  # This setting allows the following character sequences to have special meaning:
+  #
+  # * `\\r` (backslash "r") - means carriage return (ASCII 0x0D)
+  # * `\\n` (backslash "n") - means newline (ASCII 0x0A)
+  config :delimiter, :validate => :string
 
   HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']
 
@@ -67,6 +86,13 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   public
   def initialize(params={})
     super(params)
+    if @delimiter
+      # Logstash configuration doesn't have built-in support for escaping,
+      # so we implement it here. Feature discussion for escaping is here:
+      #   https://github.com/elastic/logstash/issues/1645
+      @delimiter = @delimiter.gsub("\\r", "\r").gsub("\\n", "\n")
+      @buffer = FileWatch::BufferedTokenizer.new(@delimiter)
+    end
   end
 
   private
@@ -76,7 +102,17 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   end
 
   public
-  def decode(data)
+  def decode(data, &block)
+    if @delimiter
+      @buffer.extract(data).each do |line|
+        handle(line, &block)
+      end
+    else
+      handle(data, &block)
+    end
+  end
+
+  def handle(data, &block)
     # Strip any quotations at the start and end, flex connectors seem to send this
     if data[0] == "\""
       data = data[1..-2]
@@ -172,7 +208,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
 
     # :sev is deprecated and therefore only considered if :severity equals the default setting or is invalid
     severity = sanitize_severity(event, @severity)
-    if severity == self.class.get_config["severity"][:default]
+    if severity == self.class.get_config["severity"][:default] && @sev
       # Use deprecated setting sev
       severity = sanitize_severity(event, @sev)
     end
@@ -181,7 +217,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     header = ["CEF:0", vendor, product, version, signature, name, severity].join("|")
     values = @fields.map {|fieldname| get_value(fieldname, event)}.compact.join(" ")
 
-    @on_event.call(event, "#{header}|#{values}\n")
+    @on_event.call(event, "#{header}|#{values}#{@delimiter}")
   end
 
   private

data/logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '4.0.0'
+  s.version         = '4.1.0'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "CEF codec to parse and encode CEF formated logs"

data/spec/codecs/cef_spec.rb

@@ -14,6 +14,22 @@ describe LogStash::Codecs::CEF do
 
     let(:results)   { [] }
 
+    context "with delimiter set" do 
+      # '\r\n' in single quotes to simulate the real input from a config
+      # containing \r\n as 4-character sequence in the config:
+      #
+      #   delimiter => "\r\n"
+      #
+      # Related: https://github.com/elastic/logstash/issues/1645
+      subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
+
+      it "should append the delimiter to the result" do
+        codec.on_event { |data, newdata| results << newdata }
+        codec.encode(LogStash::Event.new({}))
+        expect(results.first).to end_with("\r\n")
+      end
+    end
+
     it "should not fail if fields is nil" do
       codec.on_event{|data, newdata| results << newdata}
       event = LogStash::Event.new("foo" => "bar")
@@ -328,6 +344,32 @@ describe LogStash::Codecs::CEF do
       insist { e.get('severity') } == "10"
     end
 
+    context "with delimiter set" do 
+      # '\r\n' in single quotes to simulate the real input from a config
+      # containing \r\n as 4-character sequence in the config:
+      #
+      #   delimiter => "\r\n"
+      #
+      # Related: https://github.com/elastic/logstash/issues/1645
+      subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
+
+      it "should parse on the delimiter " do
+        subject.decode(message) do |e|
+          raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
+        end
+
+        event = false;
+        subject.decode("\r\n") do |e|
+          validate(e)
+          insist { e.get("deviceVendor") } == "security"
+          insist { e.get("deviceProduct") } == "threatmanager"
+          event = true
+        end
+
+        expect(event).to be_truthy
+      end
+    end
+
     it "should parse the cef headers" do
       subject.decode(message) do |e|
         validate(e)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-cef
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.1.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-17 00:00:00.000000000 Z
+date: 2016-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement