logstash-codec-cef 3.0.0-java → 4.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a841a39cd9b28c0794c9cb08e8295c5eea3bb521
-  data.tar.gz: 9d6a5196496f277d25bf0064da05499c8240f2e1
+  metadata.gz: c46dcaf722e0c1935b66dfcf7a7d71e3bcc51844
+  data.tar.gz: 160e4b282a46867cbdcada9e3ca1f674ade9343f
 SHA512:
-  metadata.gz: 1517164e07a5e5d89d1a2a768b9d0dd676c3175cbf6fbe40742e41d9fd8f163f24d15605dae02d1c2e241be9815e8de6fc6e617812fbb076038512f85af82a11
-  data.tar.gz: 32de38bce405918a55b762161d1d1ebd603660caeeeb5b0f86fbce8f5ee9b7320f49b961ec13b6714163a75201582d872238b6198fa574f1abe547b682e3dd43
+  metadata.gz: b0f028820eb77468b5cb98c2e9a7ebad5205119b9da6fd43b2bd0b193ede44279593cc901a35713c1cf1e7c264e9d2c5bf601cbfe82d79c5a51b4d8086c44a59
+  data.tar.gz: bb86b4781182b84949b29bdcd5659b52f5e3a11c0211a5f2a984051ff182d9cd84c2879ba671bbcf18ab529fc2086399d35cb87e334de6827277f88c65fb2773

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.0.0
+ - Implements the dictionary translation for abbreviated CEF field names from chapter Chapter 2: ArcSight Extension Dictionary    page 3 of 39 [CEF specification](https://protect724.hp.com/docs/DOC-1072).
+ - add `_cefparsefailure` tag on failed decode
+
 ## 3.0.0
  - breaking: Updated plugin to use new Java Event APIs
 
@@ -15,7 +19,6 @@
  - Config option `sev` is deprecated, use `severity` instead.
 
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
-

data/CONTRIBUTORS

@@ -11,6 +11,7 @@ Contributors:
 * Jordan Sissel (jordansissel)
 * João Duarte (jsvd)
 * Nick Ethier (nickethier)
+* Nicholas Lim (nich07as)
 * Pete Fritchman (fetep)
 * Pier-Hugues Pellerin (ph)
 * Karl Stoney (Stono)

data/lib/logstash/codecs/cef.rb

@@ -5,6 +5,9 @@ require "json"
 # Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
 # Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
 # https://protect724.hp.com/servlet/JiveServlet/downloadBody/1072-102-6-4697/CommonEventFormat.pdf
+#
+# If this codec receives a payload from an input that is not a valid CEF message, then it will
+# produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
 class LogStash::Codecs::CEF < LogStash::Codecs::Base
   config_name "cef"
 
@@ -49,8 +52,18 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   # Fields to be included in CEV extension part as key/value pairs
   config :fields, :validate => :array, :default => []
 
-  HEADER_FIELDS = ['cef_version','cef_vendor','cef_product','cef_device_version','cef_sigid','cef_name','cef_severity']
-  
+  # Set this flag if you want to have both v1 and v2 fields indexed at the same time. Note that this option will increase 
+  # the index size and data stored in outputs like Elasticsearch
+  # This option is available to ease transition to new schema
+  config :deprecated_v1_fields, :validate => :boolean, :default => false, :deprecated => "This setting is being deprecated"
+
+  HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']
+
+  # Translating and flattening the CEF extensions with known field names as documented in the Common Event Format whitepaper
+  MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint4Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNTDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "receiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime" }
+
+  DEPRECATED_HEADER_FIELDS = ['cef_version','cef_vendor','cef_product','cef_device_version','cef_sigid','cef_name','cef_severity']
+
   public
   def initialize(params={})
     super(params)
@@ -77,6 +90,12 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     # TODO: To solve all unescaping cases, regex is not suitable. A little parse should be written.
     split_data = data.split /(?<=[^\\]\\\\)[\|]|(?<!\\)[\|]/
 
+    # To be invoked when config settings is set to TRUE for V1 field names (cef_ext.<fieldname>) the following code might be removed in upcoming Codec revision
+    if deprecated_v1_fields
+      handle_v1_fields(event, split_data)
+    end
+
+    # To be invoked with default config settings to utilise the new field name formatting and flatten out the JSON document
     # Store header fields
     HEADER_FIELDS.each_with_index do |field_name, index|
       store_header_field(event,field_name,split_data[index])
@@ -85,14 +104,14 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     message = split_data[HEADER_FIELDS.size..-1].join('|')
 
     # Try and parse out the syslog header if there is one
-    if event.get('cef_version').include? ' '
-      split_cef_version= event.get('cef_version').rpartition(' ')
+    if event.get('cefVersion').include? ' '
+      split_cef_version= event.get('cefVersion').rpartition(' ')
       event.set('syslog', split_cef_version[0])
-      event.set('cef_version',split_cef_version[2])
+      event.set('cefVersion',split_cef_version[2])
     end
 
     # Get rid of the CEF bit in the version
-    event.set('cef_version', event.get('cef_version').sub(/^CEF:/, ''))
+    event.set('cefVersion', event.get('cefVersion').sub(/^CEF:/, ''))
 
     # Strip any whitespace from the message
     if not message.nil? and message.include? '='
@@ -100,20 +119,36 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
 
       # If the last KVP has no value, add an empty string, this prevents hash errors below
       if message.end_with?('=')
-        message=message + ' ' unless message.end_with?('\=')
+        message = message + ' ' unless message.end_with?('\=')
       end
 
-      # Now parse the key value pairs into it
-      extensions = {}
-      message = message.split(/ ([\w\.]+)=/)
-      key, value = message.shift.split('=', 2)
-      extensions[key] = value.gsub(/\\=/, '=').gsub(/\\\\/, '\\')
-      Hash[*message].each{ |k, v| extensions[k] = v }
-      # And save the new has as the extensions
-      event.set('cef_ext', extensions)
+      # Insert custom delimiter to separate key-value pairs, to which some values will contain special characters
+      # This separator '|^^^' os tested to be unique
+      message = message.gsub((/\s+(\w+=)/),'|^^^\1')
+
+      # This portion strips out the additional fields from the CEF logs, if needed, the ArcSight connectors will need to map it accordingly
+      # Also implemented to safeguard the {dot} notations in ES versions below 2.4x
+      message = message.gsub((/ (ad\.\w+.*\]\=[^|^^^]+)/),'')
+      message = message.split('|^^^')
+
+      # Replaces the '=' with '***' to avoid conflict with strings with HTML content namely key-value pairs where the values contain HTML strings
+      # Example : requestUrl = http://<testdomain>:<port>?query=A
+      for i in 0..message.length-1
+        message[i] = message[i].sub(/\=/, "***")
+        message[i] = message[i].gsub(/\\=/, '=').gsub(/\\\\/, '\\')
+      end
+
+      message = message.map {|s| k, v = s.split('***'); "#{MAPPINGS[k] || k }=#{v}"}
+      message = message.each_with_object({}) do |k|
+        key, value = k.split(/\s*=\s*/,2)
+        event.set(key, value)
+      end
     end
 
     yield event
+  rescue => e
+    @logger.error("Failed to decode CEF payload. Generating failure event with payload in message field.", :error => e.message, :backtrace => e.backtrace, :data => data)
+    yield LogStash::Event.new("message" => data, "tags" => ["_cefparsefailure"])
   end
 
   public
@@ -230,5 +265,44 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   rescue TypeError, ArgumentError
     false
   end
+  
+  def handle_v1_fields(event, split_data)
+    # Store header fields
+    DEPRECATED_HEADER_FIELDS.each_with_index do |field_name, index|
+      store_header_field(event,field_name,split_data[index])
+    end
+    #Remainder is message
+    message = split_data[DEPRECATED_HEADER_FIELDS.size..-1].join('|')
+
+    # Try and parse out the syslog header if there is one
+    if event.get('cef_version').include? ' '
+      split_cef_version= event.get('cef_version').rpartition(' ')
+      event.set('syslog', split_cef_version[0])
+      event.set('cef_version',split_cef_version[2])
+    end
+
+    # Get rid of the CEF bit in the version
+    event.set('cef_version', event.get('cef_version').sub(/^CEF:/, ''))
+
+    # Strip any whitespace from the message
+    if not message.nil? and message.include? '='
+      message = message.strip
+
+      # If the last KVP has no value, add an empty string, this prevents hash errors below
+      if message.end_with?('=')
+        message=message + ' ' unless message.end_with?('\=')
+      end
+
+      # Now parse the key value pairs into it
+      extensions = {}
+      message = message.split(/ ([\w\.]+)=/)
+      key, value = message.shift.split('=', 2)
+      extensions[key] = value.gsub(/\\=/, '=').gsub(/\\\\/, '\\')
+      Hash[*message].each{ |k, v| extensions[k] = v }
+      # And save the new has as the extensions
+      event.set('cef_ext', extensions)
+    end
+
+  end  
 
 end

data/logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '3.0.0'
+  s.version         = '4.0.0'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "CEF codec to parse and encode CEF formated logs"

data/spec/codecs/cef_spec.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/cef"
 require "logstash/event"
@@ -320,6 +319,193 @@ describe LogStash::Codecs::CEF do
   context "#decode" do
     let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
 
+    def validate(e)
+      insist { e.is_a?(LogStash::Event) }
+      insist { e.get('cefVersion') } == "0"
+      insist { e.get('deviceVersion') } == "1.0"
+      insist { e.get('deviceEventClassId') } == "100"
+      insist { e.get('name') } == "trojan successfully stopped"
+      insist { e.get('severity') } == "10"
+    end
+
+    it "should parse the cef headers" do
+      subject.decode(message) do |e|
+        validate(e)
+        insist { e.get("deviceVendor") } == "security"
+        insist { e.get("deviceProduct") } == "threatmanager"
+      end
+    end
+
+    it "should parse the cef body" do
+      subject.decode(message) do |e|
+        insist { e.get("sourceAddress")} == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("sourcePort") } == "1232"
+      end
+    end
+
+    let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+    it "should be OK with missing CEF headers (multiple pipes in sequence)" do
+      subject.decode(missing_headers) do |e|
+        validate(e)
+        insist { e.get("deviceVendor") } == ""
+        insist { e.get("deviceProduct") } == ""
+      end
+    end
+
+    let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+    it "should strip leading whitespace from the message" do
+      subject.decode(leading_whitespace) do |e|
+        validate(e)
+      end
+    end
+
+    let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
+    it "should be OK with escaped pipes in the message" do
+      subject.decode(escaped_pipes) do |e|
+        insist { e.get("moo") } == 'this\|has an escaped pipe'
+      end
+    end
+
+    let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
+    it "should be OK with not escaped pipes in the message" do
+      subject.decode(pipes_in_message) do |e|
+        insist { e.get("moo") } == 'this|has an pipe'
+      end
+    end
+
+    let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
+    it "should be OK with equal in the message" do
+      subject.decode(equal_in_message) do |e|
+        insist { e.get("moo") } == 'this =has = equals='
+      end
+    end
+
+    let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
+    it "should be OK with escaped backslash in the headers" do
+      subject.decode(escaped_backslash_in_header) do |e|
+        insist { e.get("cefVersion") } == '0'
+        insist { e.get("deviceVendor") } == 'secu\\rity'
+        insist { e.get("deviceProduct") } == 'threat\\manager'
+        insist { e.get("deviceVersion") } == '1.\\0'
+        insist { e.get("deviceEventClassId") } == '10\\0'
+        insist { e.get("name") } == 'tro\\jan successfully stopped'
+        insist { e.get("severity") } == '\\10'
+      end
+    end
+
+    let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
+    it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
+      subject.decode(escaped_backslash_in_header_edge_case) do |e|
+        validate(e)
+        insist { e.get("deviceVendor") } == 'security\\|'
+        insist { e.get("deviceProduct") } == 'threatmanager\\'
+      end
+    end
+
+    let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
+    it "should be OK with escaped pipes in the headers" do
+      subject.decode(escaped_pipes_in_header) do |e|
+        insist { e.get("cefVersion") } == '0'
+        insist { e.get("deviceVendor") } == 'secu|rity'
+        insist { e.get("deviceProduct") } == 'threatmanager|'
+        insist { e.get("deviceVersion") } == '1.|0'
+        insist { e.get("deviceEventClassId") } == '10|0'
+        insist { e.get("name") } == 'tro|jan successfully stopped'
+        insist { e.get("severity") } == '|10'
+      end
+    end
+
+    let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
+    it "should be OK with backslashs in the message" do
+      subject.decode(backslash_in_message) do |e|
+        insist { e.get("moo") } == 'this \\has \\ backslashs\\'
+      end
+    end
+
+    let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
+    it "should be OK with equal in the headers" do
+      subject.decode(equal_in_header) do |e|
+        validate(e)
+        insist { e.get("deviceProduct") } == "threatmanager=equal"
+      end
+    end
+
+    let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192  dst=12.121.122.82  spt=1232'}
+    it "should be OK to have one or more spaces between keys" do
+      subject.decode(spaces_in_between_keys) do |e|
+        validate(e)
+        insist { e.get("sourceAddress") } == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("sourcePort") } == "1232"
+      end
+    end
+
+    let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82  spt=1232 dproc=InternetExplorer x.x.x.x'}
+    it "should be OK to have one or more spaces in values" do
+      subject.decode(allow_spaces_in_values) do |e|
+        validate(e)
+        insist { e.get("sourceAddress") } == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("sourcePort") } == "1232"
+        insist { e.get("destinationProcessName") } == "InternetExplorer x.x.x.x"
+      end
+    end
+
+    let (:trim_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
+    it "should remove ad.fields" do
+      subject.decode(trim_additional_fields_with_dot_notations) do |e|
+        validate(e)
+        insist { e.get("sourceAddress") } == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("ad.field[0]") } == nil
+        insist { e.get("ad.name[1]") } == nil
+      end
+    end
+
+    let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
+    it "should remove ad.fields" do
+      subject.decode(preserve_unmatched_key_mappings) do |e|
+        validate(e)
+        insist { e.get("sourceAddress") } == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("new_key_by_device") } == "new_values here"
+      end
+    end
+
+    let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200'}
+    it "should translate most known abbreviated CEF field names" do
+      subject.decode(translate_abbreviated_cef_fields) do |e|
+        validate(e)
+        insist { e.get("sourceAddress") } == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("transportProtocol") } == "TCP"
+        insist { e.get("sourceHostName") } == "source.host.name"
+        insist { e.get("destinationHostName") } == "destination.host.name"
+        insist { e.get("sourcePort") } == "11024"
+        insist { e.get("destinationPort") } == "9200"
+      end
+    end
+
+    let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+    it "Should detect headers before CEF starts" do
+      subject.decode(syslog) do |e|
+        validate(e)
+        insist { e.get('syslog') } == 'Syslogdate Sysloghost'
+      end
+    end
+  end
+  
+  context "decode with deprecated version option" do
+    let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+    let(:options) {
+      {
+        "deprecated_v1_fields" => true
+      }
+    }
+    
+    subject(:codec) { LogStash::Codecs::CEF.new(options) }
+    
     def validate(e) 
       insist { e.is_a?(LogStash::Event) }
       insist { e.get('cef_version') } == "0"
@@ -327,6 +513,11 @@ describe LogStash::Codecs::CEF do
       insist { e.get('cef_sigid') } == "100"
       insist { e.get('cef_name') } == "trojan successfully stopped"
       insist { e.get('cef_severity') } == "10"
+      insist { e.get('cefVersion') } == "0"
+      insist { e.get('deviceVersion') } == "1.0"
+      insist { e.get('deviceEventClassId') } == "100"
+      insist { e.get('name') } == "trojan successfully stopped"
+      insist { e.get('severity') } == "10"
     end
 
     it "should parse the cef headers" do
@@ -335,6 +526,8 @@ describe LogStash::Codecs::CEF do
         ext = e.get('cef_ext')
         insist { e.get("cef_vendor") } == "security"
         insist { e.get("cef_product") } == "threatmanager"
+        insist { e.get("deviceVendor") } == "security"
+        insist { e.get("deviceProduct") } == "threatmanager"
       end
     end
 
@@ -344,6 +537,9 @@ describe LogStash::Codecs::CEF do
         insist { ext['src'] } == "10.0.0.192"
         insist { ext['dst'] } == "12.121.122.82"
         insist { ext['spt'] } == "1232"
+        insist { e.get("sourceAddress")} == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("sourcePort") } == "1232"
       end
     end
 
@@ -361,6 +557,8 @@ describe LogStash::Codecs::CEF do
         validate(e)
         insist { e.get("cef_vendor") } == ""
         insist { e.get("cef_product") } == ""
+        insist { e.get("deviceVendor") } == ""
+        insist { e.get("deviceProduct") } == ""
       end 
     end
 
@@ -427,6 +625,13 @@ describe LogStash::Codecs::CEF do
         insist { e.get("cef_sigid") } == '10|0'
         insist { e.get("cef_name") } == 'tro|jan successfully stopped'
         insist { e.get("cef_severity") } == '|10'
+        insist { e.get("cefVersion") } == '0'
+        insist { e.get("deviceVendor") } == 'secu|rity'
+        insist { e.get("deviceProduct") } == 'threatmanager|'
+        insist { e.get("deviceVersion") } == '1.|0'
+        insist { e.get("deviceEventClassId") } == '10|0'
+        insist { e.get("name") } == 'tro|jan successfully stopped'
+        insist { e.get("severity") } == '|10'
       end
     end
 
@@ -451,7 +656,16 @@ describe LogStash::Codecs::CEF do
       subject.decode(syslog) do |e|
         validate(e)
         insist { e.get('syslog') } == 'Syslogdate Sysloghost'
-      end 
+      end
+    end
+
+    context "when payload is not in CEF" do
+      let (:message) { "potatoes" }
+      it "Should detect headers before CEF starts" do
+        subject.decode(message) do |e|
+          insist { e.get('tags') } == ['_cefparsefailure']
+        end
+      end
     end
   end
 
@@ -462,24 +676,23 @@ describe LogStash::Codecs::CEF do
 
     it "should return an equal event if encoded and decoded again" do
       codec.on_event{|data, newdata| results << newdata}
-      codec.vendor = "%{cef_vendor}"
-      codec.product = "%{cef_product}"
-      codec.version = "%{cef_device_version}"
-      codec.signature = "%{cef_sigid}"
-      codec.name = "%{cef_name}"
-      codec.severity = "%{cef_severity}"
+      codec.vendor = "%{deviceVendor}"
+      codec.product = "%{deviceProduct}"
+      codec.version = "%{deviceVersion}"
+      codec.signature = "%{deviceEventClassId}"
+      codec.name = "%{name}"
+      codec.severity = "%{severity}"
       codec.fields = [ "foo" ]
-      event = LogStash::Event.new("cef_vendor" => "vendor", "cef_product" => "product", "cef_device_version" => "2.0", "cef_sigid" => "signature", "cef_name" => "name", "cef_severity" => "1", "foo" => "bar")
+      event = LogStash::Event.new("deviceVendor" => "vendor", "deviceProduct" => "product", "deviceVersion" => "2.0", "deviceEventClassId" => "signature", "name" => "name", "severity" => "1", "foo" => "bar")
       codec.encode(event)
       codec.decode(results.first) do |e|
-        expect(e.get('cef_vendor')).to be == event.get('cef_vendor')
-        expect(e.get('cef_product')).to be == event.get('cef_product')
-        expect(e.get('cef_device_version')).to be == event.get('cef_device_version')
-        expect(e.get('cef_sigid')).to be == event.get('cef_sigid')
-        expect(e.get('cef_name')).to be == event.get('cef_name')
-        expect(e.get('cef_severity')).to be == event.get('cef_severity')
-        # decode saves extensions as hash to 'cef_ext'
-        expect(e.get('[cef_ext][foo]')).to be == event.get('foo')
+        expect(e.get('deviceVendor')).to be == event.get('deviceVendor')
+        expect(e.get('deviceProduct')).to be == event.get('deviceProduct')
+        expect(e.get('deviceVersion')).to be == event.get('deviceVersion')
+        expect(e.get('deviceEventClassId')).to be == event.get('deviceEventClassId')
+        expect(e.get('name')).to be == event.get('name')
+        expect(e.get('severity')).to be == event.get('severity')
+        expect(e.get('foo')).to be == event.get('foo')
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-cef
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 4.0.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-22 00:00:00.000000000 Z
+date: 2016-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement