logstash-codec-cef 5.0.3-java → 5.0.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c1e2d59b4849c66f6d60d93c0fe03f11e330c97bedfe25280919f3651b5508c
-  data.tar.gz: 4b44ff90abb4bbb14e3a5268df6a841e9354f49ab8fef1c3dfd8ffb6798cde85
+  metadata.gz: 2d6aa2e3f0deee7e7dc16646e1803e2514f2e52e5396329c5ca8fbc6a9a11890
+  data.tar.gz: fdca34d3a6ce64552a5965a60543c97bd23629ae521f04d9d2757c3f7f5d746a
 SHA512:
-  metadata.gz: 68f97c0e0361d3b889c62f8502fb2802d24770266e0dc306ee5d327c6b3e9e3405aaf9db9c53e033b46b052bec82b3f8ec9d2df63c99869d5d8e87e1523e1f89
-  data.tar.gz: e2335c058a3d7fbbfa57e57eeb008903b4423063094d543948161864438e8fd65ea09df2a275853991c0ab15122680f8fa4cccb49504ff43fdc9693658d0db75
+  metadata.gz: b7bbb1fe5a6c5915e6c613e689a2c47ce7795c6b807c564b14b271b14adfe7b8c2f46626421acb6cc9cb45c08c7772801f9ad9bc9432bebe2eb8e2fa8e9f88b1
+  data.tar.gz: d2a29e95aaa41635b6240219714da61daff58b75e37aff39e25365120ef66c066f3936d63e6cf2c028eb53e94056d42bbf2b5dc464e7502a314eb4878970349c

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 5.0.4
+ - Fix bug in parsing headers where certain legal escape sequences could cause non-escaped pipe characters to be ignored.
+ - Fix bug in parsing extension values where a legal unescaped space in a field's value could be interpreted as a field separator (#54)
+ - Add explicit handling for extension key names that use array-like syntax that isn't legal with the strict-mode field-reference parser (e.g., `fieldname[0]` becomes `[fieldname][0]`).
+
 ## 5.0.3
  - Fix handling of higher-plane UTF-8 characters in message body
 

data/lib/logstash/codecs/cef.rb

@@ -74,9 +74,132 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']
 
   # Translating and flattening the CEF extensions with known field names as documented in the Common Event Format whitepaper
-  MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2Label", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint3Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNtDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "outcome" => "eventOutcome", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "deviceReceiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime", "amac" => "agentMacAddress" }
-
-  DEPRECATED_HEADER_FIELDS = ['cef_version','cef_vendor','cef_product','cef_device_version','cef_sigid','cef_name','cef_severity']
+  MAPPINGS = {
+      "act" => "deviceAction",
+      "app" => "applicationProtocol",
+      "c6a1" => "deviceCustomIPv6Address1",
+      "c6a1Label" => "deviceCustomIPv6Address1Label",
+      "c6a2" => "deviceCustomIPv6Address2",
+      "c6a2Label" => "deviceCustomIPv6Address2Label",
+      "c6a3" => "deviceCustomIPv6Address3",
+      "c6a3Label" => "deviceCustomIPv6Address3Label",
+      "c6a4" => "deviceCustomIPv6Address4",
+      "c6a4Label" => "deviceCustomIPv6Address4Label",
+      "cat" => "deviceEventCategory",
+      "cfp1" => "deviceCustomFloatingPoint1",
+      "cfp1Label" => "deviceCustomFloatingPoint1Label",
+      "cfp2" => "deviceCustomFloatingPoint2",
+      "cfp2Label" => "deviceCustomFloatingPoint2Label",
+      "cfp3" => "deviceCustomFloatingPoint3",
+      "cfp3Label" => "deviceCustomFloatingPoint3Label",
+      "cfp4" => "deviceCustomFloatingPoint4",
+      "cfp4Label" => "deviceCustomFloatingPoint4Label",
+      "cn1" => "deviceCustomNumber1",
+      "cn1Label" => "deviceCustomNumber1Label",
+      "cn2" => "deviceCustomNumber2",
+      "cn2Label" => "deviceCustomNumber2Label",
+      "cn3" => "deviceCustomNumber3",
+      "cn3Label" => "deviceCustomNumber3Label",
+      "cnt" => "baseEventCount",
+      "cs1" => "deviceCustomString1",
+      "cs1Label" => "deviceCustomString1Label",
+      "cs2" => "deviceCustomString2",
+      "cs2Label" => "deviceCustomString2Label",
+      "cs3" => "deviceCustomString3",
+      "cs3Label" => "deviceCustomString3Label",
+      "cs4" => "deviceCustomString4",
+      "cs4Label" => "deviceCustomString4Label",
+      "cs5" => "deviceCustomString5",
+      "cs5Label" => "deviceCustomString5Label",
+      "cs6" => "deviceCustomString6",
+      "cs6Label" => "deviceCustomString6Label",
+      "dhost" => "destinationHostName",
+      "dmac" => "destinationMacAddress",
+      "dntdom" => "destinationNtDomain",
+      "dpid" => "destinationProcessId",
+      "dpriv" => "destinationUserPrivileges",
+      "dproc" => "destinationProcessName",
+      "dpt" => "destinationPort",
+      "dst" => "destinationAddress",
+      "duid" => "destinationUserId",
+      "duser" => "destinationUserName",
+      "dvc" => "deviceAddress",
+      "dvchost" => "deviceHostName",
+      "dvcpid" => "deviceProcessId",
+      "end" => "endTime",
+      "fname" => "fileName",
+      "fsize" => "fileSize",
+      "in" => "bytesIn",
+      "msg" => "message",
+      "out" => "bytesOut",
+      "outcome" => "eventOutcome",
+      "proto" => "transportProtocol",
+      "request" => "requestUrl",
+      "rt" => "deviceReceiptTime",
+      "shost" => "sourceHostName",
+      "smac" => "sourceMacAddress",
+      "sntdom" => "sourceNtDomain",
+      "spid" => "sourceProcessId",
+      "spriv" => "sourceUserPrivileges",
+      "sproc" => "sourceProcessName",
+      "spt" => "sourcePort",
+      "src" => "sourceAddress",
+      "start" => "startTime",
+      "suid" => "sourceUserId",
+      "suser" => "sourceUserName",
+      "ahost" => "agentHost",
+      "art" => "agentReceiptTime",
+      "at" => "agentType",
+      "aid" => "agentId",
+      "_cefVer" => "cefVersion",
+      "agt" => "agentAddress",
+      "av" => "agentVersion",
+      "atz" => "agentTimeZone",
+      "dtz" => "destinationTimeZone",
+      "slong" => "sourceLongitude",
+      "slat" => "sourceLatitude",
+      "dlong" => "destinationLongitude",
+      "dlat" => "destinationLatitude",
+      "catdt" => "categoryDeviceType",
+      "mrt" => "managerReceiptTime",
+      "amac" => "agentMacAddress"
+  }
+
+  # A CEF Header is a sequence of zero or more:
+  #  - backslash-escaped pipes; OR
+  #  - backslash-escaped backslashes; OR
+  #  - non-pipe characters
+  HEADER_PATTERN = /(?:\\\||\\\\|[^|])*?/
+
+  # Cache of a scanner pattern that _captures_ a HEADER followed by an unescaped pipe
+  HEADER_SCANNER = /(#{HEADER_PATTERN})#{Regexp.quote('|')}/
+
+  # Cache of a gsub pattern that matches a backslash-escaped backslash or backslash-escaped pipe, _capturing_ the escaped character
+  HEADER_ESCAPE_CAPTURE = /\\([\\|])/
+
+  # Cache of a gsub pattern that matches a backslash-escaped backslash or backslash-escaped equals, _capturing_ the escaped character
+  EXTENSION_VALUE_ESCAPE_CAPTURE = /\\([\\=])/
+
+  # While the original CEF spec calls out that extension keys must be alphanumeric and not contain spaces,
+  # in practice many "CEF" producers like the Arcsight smart connector produce non-legal keys including underscores,
+  # commas, periods, and square-bracketed index offsets.
+  # Allow any sequence of characters that are _not_ backslashes, equals, or spaces.
+  EXTENSION_KEY_PATTERN = /[^= \\]+/
+
+  # Some CEF extension keys seen in the wild use an undocumented array-like syntax that may not be compatible with
+  # the Event API's strict-mode FieldReference parser (e.g., `fieldname[0]`).
+  # Cache of a `String#sub` pattern matching array-like syntax and capturing both the base field name and the
+  # array-indexing portion so we can convert to a valid FieldReference (e.g., `[fieldname][0]`).
+  EXTENSION_KEY_ARRAY_CAPTURE = /^([^\[\]]+)((?:\[[0-9]+\])+)$/ # '[\1]\2'
+
+  # In extensions, spaces may be included in an extension value without any escaping,
+  # so an extension value is a sequence of zero or more:
+  # - non-whitespace character; OR
+  # - runs of whitespace that are NOT followed by something that looks like a key-equals sequence
+  EXTENSION_VALUE_PATTERN = /(?:\S|\s++(?!#{EXTENSION_KEY_PATTERN}=))*/
+
+  # Cache of a scanner pattern that _captures_ extension field key/value pairs
+  EXTENSION_KEY_VALUE_SCANNER = /(#{EXTENSION_KEY_PATTERN})=(#{EXTENSION_VALUE_PATTERN})\s*/
 
   public
   def initialize(params={})
@@ -96,12 +219,6 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     end
   end
 
-  private
-  def store_header_field(event,field_name,field_data)
-    #Unescape pipes and backslash in header fields
-    event.set(field_name,field_data.gsub(/\\\|/, '|').gsub(/\\\\/, '\\')) unless field_data.nil?
-  end
-
   public
   def decode(data, &block)
     if @delimiter
@@ -128,22 +245,24 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
       data = data[1..-2]
     end
 
-    # Split by the pipes, pipes in the extension part are perfectly valid and do not need escaping
-    # The better solution for the splitting regex would be /(?<!\\(\\\\)*)[\|]/, but this
-    # gives an "SyntaxError: (RegexpError) invalid pattern in look-behind" for the variable length look behind.
-    # Therefore one edge case is not handled properly: \\| (this should split, but it does not, because the escaped \ is not recognized)
-    # TODO: To solve all unescaping cases, regex is not suitable. A little parse should be written.
-    split_data = data.split /(?<=[^\\]\\\\)[\|]|(?<!\\)[\|]/
+    # Use a scanning parser to capture the HEADER_FIELDS
+    unprocessed_data = data
+    HEADER_FIELDS.each do |field_name|
+      match_data = HEADER_SCANNER.match(unprocessed_data)
+      break if match_data.nil? # missing fields
 
-    # To be invoked when config settings is set to TRUE for V1 field names (cef_ext.<fieldname>) the following code might be removed in upcoming Codec revision
+      escaped_field_value = match_data[1]
+      next if escaped_field_value.nil?
 
-    # To be invoked with default config settings to utilise the new field name formatting and flatten out the JSON document
-    # Store header fields
-    HEADER_FIELDS.each_with_index do |field_name, index|
-      store_header_field(event,field_name,split_data[index])
+      # process legal header escape sequences
+      unescaped_field_value = escaped_field_value.gsub(HEADER_ESCAPE_CAPTURE, '\1')
+
+      event.set(field_name, unescaped_field_value)
+      unprocessed_data = match_data.post_match
     end
+
     #Remainder is message
-    message = split_data[HEADER_FIELDS.size..-1].join('|')
+    message = unprocessed_data
 
     # Try and parse out the syslog header if there is one
     if event.get('cefVersion').include? ' '
@@ -155,36 +274,21 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     # Get rid of the CEF bit in the version
     event.set('cefVersion', event.get('cefVersion').sub(/^CEF:/, ''))
 
-    # Strip any whitespace from the message
-    if not message.nil? and message.include? '='
+    # Use a scanning parser to capture the Extension Key/Value Pairs
+    if message && message.include?('=')
       message = message.strip
 
-      # If the last KVP has no value, add an empty string, this prevents hash errors below
-      if message.end_with?('=')
-        message = message + ' ' unless message.end_with?('\=')
-      end
+      message.scan(EXTENSION_KEY_VALUE_SCANNER) do |extension_field_key, raw_extension_field_value|
+        # expand abbreviated extension field keys
+        extension_field_key = MAPPINGS.fetch(extension_field_key, extension_field_key)
 
-      # Insert custom delimiter to separate key-value pairs, to which some values will contain special characters
-      # This separator '|^^^' os tested to be unique
-      message = message.gsub((/(?:(\s+(\w+\=)))/),'|^^^\2')
-
-      # Appropriately tokenizing the additional fields when ArcSight connectors are sending events using "COMPLETE" mode processing.
-      # If these fields are NOT needed, then set the ArcSight processing mode for this destination to "FASTER" or "FASTEST"
-      # Refer to ArcSight's SmartConnector user configuration guide
-      message = message.gsub((/(\s+(\w+\.[^\s]\w+[^\|\s\.\=]+\=))/),'|^^^\2')
-      message = message.split('|^^^')
-
-      # Replaces the '=' with '***' to avoid conflict with strings with HTML content namely key-value pairs where the values contain HTML strings
-      # Example : requestUrl = http://<testdomain>:<port>?query=A
-      for i in 0..message.length-1
-        message[i] = message[i].sub(/\=/, "***")
-        message[i] = message[i].gsub(/\\=/, '=').gsub(/\\\\/, '\\')
-      end
+        # convert extension field name to strict legal field_reference, fixing field names with ambiguous array-like syntax
+        extension_field_key = extension_field_key.sub(EXTENSION_KEY_ARRAY_CAPTURE, '[\1]\2') if extension_field_key.end_with?(']')
+
+        # process legal extension field value escapes
+        extension_field_value = raw_extension_field_value.gsub(EXTENSION_VALUE_ESCAPE_CAPTURE, '\1')
 
-      message = message.map {|s| k, v = s.split('***'); "#{MAPPINGS[k] || k }=#{v}"}
-      message = message.each_with_object({}) do |k|
-        key, value = k.split(/\s*=\s*/,2)
-        event.set(key, value)
+        event.set(extension_field_key, extension_field_value)
       end
     end
 
@@ -303,44 +407,4 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   rescue TypeError, ArgumentError
     false
   end
-
-  def handle_v1_fields(event, split_data)
-    # Store header fields
-    DEPRECATED_HEADER_FIELDS.each_with_index do |field_name, index|
-      store_header_field(event,field_name,split_data[index])
-    end
-    #Remainder is message
-    message = split_data[DEPRECATED_HEADER_FIELDS.size..-1].join('|')
-
-    # Try and parse out the syslog header if there is one
-    if event.get('cef_version').include? ' '
-      split_cef_version= event.get('cef_version').rpartition(' ')
-      event.set('syslog', split_cef_version[0])
-      event.set('cef_version',split_cef_version[2])
-    end
-
-    # Get rid of the CEF bit in the version
-    event.set('cef_version', event.get('cef_version').sub(/^CEF:/, ''))
-
-    # Strip any whitespace from the message
-    if not message.nil? and message.include? '='
-      message = message.strip
-
-      # If the last KVP has no value, add an empty string, this prevents hash errors below
-      if message.end_with?('=')
-        message=message + ' ' unless message.end_with?('\=')
-      end
-
-      # Now parse the key value pairs into it
-      extensions = {}
-      message = message.split(/ ([\w\.]+)=/)
-      key, value = message.shift.split('=', 2)
-      extensions[key] = value.gsub(/\\=/, '=').gsub(/\\\\/, '\\')
-      Hash[*message].each{ |k, v| extensions[k] = v }
-      # And save the new has as the extensions
-      event.set('cef_ext', extensions)
-    end
-
-  end
-
 end

data/logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '5.0.3'
+  s.version         = '5.0.4'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads the ArcSight Common Event Format (CEF)."

data/spec/codecs/cef_spec.rb

@@ -296,6 +296,49 @@ describe LogStash::Codecs::CEF do
       insist { e.get('severity') } == "10"
     end
 
+    ##
+    # Use the given codec to decode the given data, ensuring exactly one event is emitted.
+    #
+    # If a block is given, yield the resulting event to the block _outside_ of `LogStash::Codecs::CEF#decode(String)`
+    # in order to avoid mismatched-exceptions raised by RSpec triggering the codec's exception-handling.
+    #
+    # @param codec [#decode]
+    # @param data [String]
+    # @yieldparam event [Event]
+    # @yieldreturn [void]
+    # @return [Event]
+    def decode_one(codec, data)
+      events = do_decode(codec, data)
+      fail("Expected one event, got #{events.size} events: #{events.inspect}") unless events.size == 1
+      event = events.first
+
+      yield event if block_given?
+
+      event
+    end
+
+    ##
+    # Use the given codec to decode the given data, returning an Array of the resulting Events
+    #
+    # If a block is given, each event is yielded to the block _outside_ of `LogStash::Codecs::CEF#decode(String)`
+    # in order to avoid mismatched-exceptions raised by RSpec triggering the codec's exception-handling.
+    #
+    # @param codec [#decode]
+    # @param data [String]
+    # @yieldparam event [Event]
+    # @yieldreturn [void]
+    # @return [Array<Event>]
+    def do_decode(codec, data)
+      events = []
+      codec.decode(data) do |event|
+        events << event
+      end
+
+      events.each { |event| yield event } if block_given?
+
+      events
+    end
+
     context "with delimiter set" do
       # '\r\n' in single quotes to simulate the real input from a config
       # containing \r\n as 4-character sequence in the config:
@@ -306,24 +349,36 @@ describe LogStash::Codecs::CEF do
       subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
 
       it "should parse on the delimiter " do
-        subject.decode(message) do |e|
+        do_decode(subject,message) do |e|
           raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
         end
 
-        event = false;
-        subject.decode("\r\n") do |e|
+        decode_one(subject, "\r\n") do |e|
           validate(e)
           insist { e.get("deviceVendor") } == "security"
           insist { e.get("deviceProduct") } == "threatmanager"
-          event = true
         end
+      end
+    end
+
+    context 'when a CEF header ends with a pair of properly-escaped backslashes' do
+      let(:backslash) { '\\' }
+      let(:pipe) { '|' }
+      let(:message) { "CEF:0|security|threatmanager|1.0|100|double backslash" +
+                      backslash + backslash + # escaped backslash
+                      backslash + backslash + # escaped backslash
+                      "|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
 
-        expect(event).to be_truthy
+      it 'should include the backslashes unescaped' do
+        event = decode_one(subject, message)
+
+        expect(event.get('name')).to eq('double backslash' + backslash + backslash )
+        expect(event.get('severity')).to eq('10') # ensure we didn't consume the separator
       end
     end
 
     it "should parse the cef headers" do
-      subject.decode(message) do |e|
+      decode_one(subject, message) do |e|
         validate(e)
         insist { e.get("deviceVendor") } == "security"
         insist { e.get("deviceProduct") } == "threatmanager"
@@ -331,7 +386,7 @@ describe LogStash::Codecs::CEF do
     end
 
     it "should parse the cef body" do
-      subject.decode(message) do |e|
+      decode_one(subject, message) do |e|
         insist { e.get("sourceAddress")} == "10.0.0.192"
         insist { e.get("destinationAddress") } == "12.121.122.82"
         insist { e.get("sourcePort") } == "1232"
@@ -340,7 +395,7 @@ describe LogStash::Codecs::CEF do
 
     let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
     it "should be OK with missing CEF headers (multiple pipes in sequence)" do
-      subject.decode(missing_headers) do |e|
+      decode_one(subject, missing_headers) do |e|
         validate(e)
         insist { e.get("deviceVendor") } == ""
         insist { e.get("deviceProduct") } == ""
@@ -349,35 +404,50 @@ describe LogStash::Codecs::CEF do
 
     let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
     it "should strip leading whitespace from the message" do
-      subject.decode(leading_whitespace) do |e|
+      decode_one(subject, leading_whitespace) do |e|
         validate(e)
       end
     end
 
     let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
     it "should be OK with escaped pipes in the message" do
-      subject.decode(escaped_pipes) do |e|
+      decode_one(subject, escaped_pipes) do |e|
         insist { e.get("moo") } == 'this\|has an escaped pipe'
       end
     end
 
     let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
     it "should be OK with not escaped pipes in the message" do
-      subject.decode(pipes_in_message) do |e|
+      decode_one(subject, pipes_in_message) do |e|
         insist { e.get("moo") } == 'this|has an pipe'
       end
     end
 
+    # while we may see these in practice, equals MUST be escaped in the extensions per the spec.
     let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
     it "should be OK with equal in the message" do
-      subject.decode(equal_in_message) do |e|
+      decode_one(subject, equal_in_message) do |e|
         insist { e.get("moo") } == 'this =has = equals='
       end
     end
 
+    context('escaped-equals and unescaped-spaces in the extension values') do
+      let(:query_string) { 'key1=value1&key2=value3 aa.bc&key3=value4'}
+      let(:escaped_query_string) { query_string.gsub('=','\\=') }
+      let(:cef_message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|go=start now query_string=#{escaped_query_string} final=done" }
+
+      it 'captures the extension values correctly' do
+        event = decode_one(subject, cef_message)
+
+        expect(event.get('go')).to eq('start now')
+        expect(event.get('query_string')).to eq(query_string)
+        expect(event.get('final')).to eq('done')
+      end
+    end
+
     let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
     it "should be OK with escaped backslash in the headers" do
-      subject.decode(escaped_backslash_in_header) do |e|
+      decode_one(subject, escaped_backslash_in_header) do |e|
         insist { e.get("cefVersion") } == '0'
         insist { e.get("deviceVendor") } == 'secu\\rity'
         insist { e.get("deviceProduct") } == 'threat\\manager'
@@ -390,7 +460,7 @@ describe LogStash::Codecs::CEF do
 
     let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
     it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
-      subject.decode(escaped_backslash_in_header_edge_case) do |e|
+      decode_one(subject, escaped_backslash_in_header_edge_case) do |e|
         validate(e)
         insist { e.get("deviceVendor") } == 'security\\|'
         insist { e.get("deviceProduct") } == 'threatmanager\\'
@@ -399,7 +469,7 @@ describe LogStash::Codecs::CEF do
 
     let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
     it "should be OK with escaped pipes in the headers" do
-      subject.decode(escaped_pipes_in_header) do |e|
+      decode_one(subject, escaped_pipes_in_header) do |e|
         insist { e.get("cefVersion") } == '0'
         insist { e.get("deviceVendor") } == 'secu|rity'
         insist { e.get("deviceProduct") } == 'threatmanager|'
@@ -412,14 +482,14 @@ describe LogStash::Codecs::CEF do
 
     let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
     it "should be OK with backslashs in the message" do
-      subject.decode(backslash_in_message) do |e|
+      decode_one(subject, backslash_in_message) do |e|
         insist { e.get("moo") } == 'this \\has \\ backslashs\\'
       end
     end
 
     let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
     it "should be OK with equal in the headers" do
-      subject.decode(equal_in_header) do |e|
+      decode_one(subject, equal_in_header) do |e|
         validate(e)
         insist { e.get("deviceProduct") } == "threatmanager=equal"
       end
@@ -427,7 +497,7 @@ describe LogStash::Codecs::CEF do
 
     let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192  dst=12.121.122.82  spt=1232'}
     it "should be OK to have one or more spaces between keys" do
-      subject.decode(spaces_in_between_keys) do |e|
+      decode_one(subject, spaces_in_between_keys) do |e|
         validate(e)
         insist { e.get("sourceAddress") } == "10.0.0.192"
         insist { e.get("destinationAddress") } == "12.121.122.82"
@@ -437,7 +507,7 @@ describe LogStash::Codecs::CEF do
 
     let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82  spt=1232 dproc=InternetExplorer x.x.x.x'}
     it "should be OK to have one or more spaces in values" do
-      subject.decode(allow_spaces_in_values) do |e|
+      decode_one(subject, allow_spaces_in_values) do |e|
         validate(e)
         insist { e.get("sourceAddress") } == "10.0.0.192"
         insist { e.get("destinationAddress") } == "12.121.122.82"
@@ -448,26 +518,26 @@ describe LogStash::Codecs::CEF do
 
     let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
     it "should keep ad.fields" do
-      subject.decode(preserve_additional_fields_with_dot_notations) do |e|
+      decode_one(subject, preserve_additional_fields_with_dot_notations) do |e|
         validate(e)
         insist { e.get("sourceAddress") } == "10.0.0.192"
         insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("ad.field[0]") } == "field0"
-        insist { e.get("ad.name[1]") } == "new_name"
+        insist { e.get("[ad.field][0]") } == "field0"
+        insist { e.get("[ad.name][1]") } == "new_name"
         insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
-        insist { e.get("ad.Error_,Code") } == "3221225578"
+        insist { e.get('ad.Error_,Code') } == "3221225578"
         insist { e.get("additional.dotfieldName") } == "new_value"
       end
     end
 
     let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000  23041810181C0000  /CN\=random.user/OU\=User Login End-Entity  /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
     it "should correctly parse random values even with additional fields in message" do
-      subject.decode(preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
+      decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
         validate(e)
         insist { e.get("sourceAddress") } == "10.0.0.192"
         insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("ad.field[0]") } == "field0"
-        insist { e.get("ad.name[1]") } == "new_name"
+        insist { e.get("[ad.field][0]") } == "field0"
+        insist { e.get("[ad.name][1]") } == "new_name"
         insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
         insist { e.get("ad.Error_,Code") } == "3221225578"
         insist { e.get("additional.dotfieldName") } == "new_value"
@@ -477,7 +547,7 @@ describe LogStash::Codecs::CEF do
 
     let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
     it "should preserve unmatched key mappings" do
-      subject.decode(preserve_unmatched_key_mappings) do |e|
+      decode_one(subject, preserve_unmatched_key_mappings) do |e|
         validate(e)
         insist { e.get("sourceAddress") } == "10.0.0.192"
         insist { e.get("destinationAddress") } == "12.121.122.82"
@@ -487,7 +557,7 @@ describe LogStash::Codecs::CEF do
 
     let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
     it "should translate most known abbreviated CEF field names" do
-      subject.decode(translate_abbreviated_cef_fields) do |e|
+      decode_one(subject, translate_abbreviated_cef_fields) do |e|
         validate(e)
         insist { e.get("sourceAddress") } == "10.0.0.192"
         insist { e.get("destinationAddress") } == "12.121.122.82"
@@ -503,7 +573,7 @@ describe LogStash::Codecs::CEF do
 
     let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
     it "Should detect headers before CEF starts" do
-      subject.decode(syslog) do |e|
+      decode_one(subject, syslog) do |e|
         validate(e)
         insist { e.get('syslog') } == 'Syslogdate Sysloghost'
       end
@@ -522,7 +592,7 @@ describe LogStash::Codecs::CEF do
         context "externally encoded as #{external_encoding}" do
           let(:message) { super().force_encoding(external_encoding) }
           it 'should keep the higher-plane characters' do
-            subject.decode(message.dup) do |event|
+            decode_one(subject, message.dup) do |event|
               validate(event)
               insist { event.get("target") } == "aaaaaああああaaaa"
               insist { event.get("target").encoding } == Encoding::UTF_8
@@ -535,7 +605,7 @@ describe LogStash::Codecs::CEF do
     context 'non-UTF-8 message' do
       let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
       it 'should emit message unparsed with _cefparsefailure tag' do
-        subject.decode(message.dup) do |event|
+        decode_one(subject, message.dup) do |event|
           insist { event.get("message").bytes.to_a } == message.bytes.to_a
           insist { event.get("tags") } == ['_cefparsefailure']
         end
@@ -546,7 +616,7 @@ describe LogStash::Codecs::CEF do
       subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
 
       it "should return the raw message in field message_raw" do
-        subject.decode(message.dup) do |e|
+        decode_one(subject, message.dup) do |e|
           validate(e)
           insist { e.get("message_raw") } == message
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-cef
 version: !ruby/object:Gem::Version
-  version: 5.0.3
+  version: 5.0.4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-19 00:00:00.000000000 Z
+date: 2018-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement