logstash-codec-cef 4.1.0-java → 4.1.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f01e9a10dba1242f1db5e742622aa2b919a4e611
-  data.tar.gz: 0381084bd82fe355d71ee4b09da90042cc5d35b4
+  metadata.gz: 4970c9754e3a998c7768dfd91127908d00754c2d
+  data.tar.gz: c89d21a60488b04b8f305346b35715bf398b22b2
 SHA512:
-  metadata.gz: 8eb490d24e26857673c34e6deb1b99a8b3be90ed168b015024aa4ad80d9f2565f1cd2b7f99fd761b5a300cf3de616af8d1fb75f2fb5c620930b7136ede4542d4
-  data.tar.gz: 3d64f1093864b14fc3a58f15cc19a3e5ab1942b286d7f35dda70071efd1a992db5a362595bad6fa1345a92bda367e3a43b812dc4af16b4012d2d639bdb534a70
+  metadata.gz: 1ddba453c75fec89baf0c374663721f20df6edfa054f1ff8b904e75cd95da4db92d1d6bd20771de5d1f6ba8d4bae1e7a5f15068cc250568a718265129a5b9802
+  data.tar.gz: e1132893ea01846719b888fe1f83c0d79bd1f7c6a1600ccbd6c6f6aa996915069b8eeb6f4bfae90aeb66d7f3f7281fef031bba8df4d161a98685eb6dab143923

data/CHANGELOG.md

@@ -1,8 +1,15 @@
+## 4.1.2
+ - added mapping for outcome = eventOutcome from CEF whitepaper (ref:p26/39)
+
+## 4.1.1
+ - changed rt from receiptTime to deviceReceiptTime (ref:p27/39)
+ - changed tokenizer to include additional fields (ad.fieldname)
+
 ## 4.1.0
  - Add `delimiter` setting. This allows the decoder to be used with inputs like the TCP input where event delimiters are used.
 
 ## 4.0.0
- - Implements the dictionary translation for abbreviated CEF field names from chapter Chapter 2: ArcSight Extension Dictionary    page 3 of 39 [CEF specification](https://protect724.hp.com/docs/DOC-1072).
+ - Implements the dictionary translation for abbreviated CEF field names from chapter Chapter 2: ArcSight Extension Dictionary page 3 of 39 [CEF specification](https://protect724.hp.com/docs/DOC-1072).
  - add `_cefparsefailure` tag on failed decode
 
 ## 3.0.0

data/lib/logstash/codecs/cef.rb

@@ -53,7 +53,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   # Fields to be included in CEV extension part as key/value pairs
   config :fields, :validate => :array, :default => []
 
-  # Set this flag if you want to have both v1 and v2 fields indexed at the same time. Note that this option will increase 
+  # Set this flag if you want to have both v1 and v2 fields indexed at the same time. Note that this option will increase
   # the index size and data stored in outputs like Elasticsearch
   # This option is available to ease transition to new schema
   config :deprecated_v1_fields, :validate => :boolean, :deprecated => "This setting is being deprecated"
@@ -66,7 +66,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   #     input {
   #       tcp {
   #         codec => cef { delimiter => "\r\n" }
-  #         # ... 
+  #         # ...
   #       }
   #     }
   #
@@ -79,7 +79,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']
 
   # Translating and flattening the CEF extensions with known field names as documented in the Common Event Format whitepaper
-  MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint4Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNTDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "receiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime" }
+  MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint4Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNTDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "outcome" => "eventOutcome", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "deviceReceiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime" }
 
   DEPRECATED_HEADER_FIELDS = ['cef_version','cef_vendor','cef_product','cef_device_version','cef_sigid','cef_name','cef_severity']
 
@@ -160,11 +160,12 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
 
       # Insert custom delimiter to separate key-value pairs, to which some values will contain special characters
       # This separator '|^^^' os tested to be unique
-      message = message.gsub((/\s+(\w+=)/),'|^^^\1')
+      message = message.gsub((/(?:(\s+(\w+\=)))/),'|^^^\2')
 
-      # This portion strips out the additional fields from the CEF logs, if needed, the ArcSight connectors will need to map it accordingly
-      # Also implemented to safeguard the {dot} notations in ES versions below 2.4x
-      message = message.gsub((/ (ad\.\w+.*\]\=[^|^^^]+)/),'')
+      # Appropriately tokenizing the additional fields when ArcSight connectors are sending events using "COMPLETE" mode processing.
+      # If these fields are NOT needed, then set the ArcSight processing mode for this destination to "FASTER" or "FASTEST"
+      # Refer to ArcSight's SmartConnector user configuration guide
+      message = message.gsub((/(\s+(\w+\.[^\s]\w+[^\|\s\.\=]+\=))/),'|^^^\2')
       message = message.split('|^^^')
 
       # Replaces the '=' with '***' to avoid conflict with strings with HTML content namely key-value pairs where the values contain HTML strings
@@ -301,7 +302,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   rescue TypeError, ArgumentError
     false
   end
-  
+
   def handle_v1_fields(event, split_data)
     # Store header fields
     DEPRECATED_HEADER_FIELDS.each_with_index do |field_name, index|
@@ -339,6 +340,6 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
       event.set('cef_ext', extensions)
     end
 
-  end  
+  end
 
 end

data/logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '4.1.0'
+  s.version         = '4.1.2'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "CEF codec to parse and encode CEF formated logs"
@@ -25,4 +25,3 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
 end
-

data/spec/codecs/cef_spec.rb

@@ -14,7 +14,7 @@ describe LogStash::Codecs::CEF do
 
     let(:results)   { [] }
 
-    context "with delimiter set" do 
+    context "with delimiter set" do
       # '\r\n' in single quotes to simulate the real input from a config
       # containing \r\n as 4-character sequence in the config:
       #
@@ -344,7 +344,7 @@ describe LogStash::Codecs::CEF do
       insist { e.get('severity') } == "10"
     end
 
-    context "with delimiter set" do 
+    context "with delimiter set" do
       # '\r\n' in single quotes to simulate the real input from a config
       # containing \r\n as 4-character sequence in the config:
       #
@@ -494,19 +494,37 @@ describe LogStash::Codecs::CEF do
       end
     end
 
-    let (:trim_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
-    it "should remove ad.fields" do
-      subject.decode(trim_additional_fields_with_dot_notations) do |e|
+    let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
+    it "should keep ad.fields" do
+      subject.decode(preserve_additional_fields_with_dot_notations) do |e|
         validate(e)
         insist { e.get("sourceAddress") } == "10.0.0.192"
         insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("ad.field[0]") } == nil
-        insist { e.get("ad.name[1]") } == nil
+        insist { e.get("ad.field[0]") } == "field0"
+        insist { e.get("ad.name[1]") } == "new_name"
+        insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
+        insist { e.get("ad.Error_,Code") } == "3221225578"
+        insist { e.get("additional.dotfieldName") } == "new_value"
+      end
+    end
+
+    let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000  23041810181C0000  /CN\=random.user/OU\=User Login End-Entity  /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
+    it "should correctly parse random values even with additional fields in message" do
+      subject.decode(preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
+        validate(e)
+        insist { e.get("sourceAddress") } == "10.0.0.192"
+        insist { e.get("destinationAddress") } == "12.121.122.82"
+        insist { e.get("ad.field[0]") } == "field0"
+        insist { e.get("ad.name[1]") } == "new_name"
+        insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
+        insist { e.get("ad.Error_,Code") } == "3221225578"
+        insist { e.get("additional.dotfieldName") } == "new_value"
+        insist { e.get("deviceCustomString4") } == "401 random.user Admin 0 23041A10181C0000  23041810181C0000  /CN\=random.user/OU\=User Login End-Entity  /CN\=TEST/OU\=Login CA TEST 34"
       end
     end
 
     let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
-    it "should remove ad.fields" do
+    it "should preserve unmatched key mappings" do
       subject.decode(preserve_unmatched_key_mappings) do |e|
         validate(e)
         insist { e.get("sourceAddress") } == "10.0.0.192"
@@ -515,7 +533,7 @@ describe LogStash::Codecs::CEF do
       end
     end
 
-    let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200'}
+    let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success'}
     it "should translate most known abbreviated CEF field names" do
       subject.decode(translate_abbreviated_cef_fields) do |e|
         validate(e)
@@ -526,6 +544,7 @@ describe LogStash::Codecs::CEF do
         insist { e.get("destinationHostName") } == "destination.host.name"
         insist { e.get("sourcePort") } == "11024"
         insist { e.get("destinationPort") } == "9200"
+        insist { e.get("eventOutcome") } == "Success"
       end
     end
 
@@ -537,7 +556,7 @@ describe LogStash::Codecs::CEF do
       end
     end
   end
-  
+
   context "decode with deprecated version option" do
     let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
     let(:options) {
@@ -545,10 +564,10 @@ describe LogStash::Codecs::CEF do
         "deprecated_v1_fields" => true
       }
     }
-    
+
     subject(:codec) { LogStash::Codecs::CEF.new(options) }
-    
-    def validate(e) 
+
+    def validate(e)
       insist { e.is_a?(LogStash::Event) }
       insist { e.get('cef_version') } == "0"
       insist { e.get('cef_device_version') } == "1.0"
@@ -590,7 +609,7 @@ describe LogStash::Codecs::CEF do
       subject.decode(no_ext) do |e|
         validate(e)
         insist { e.get("cef_ext") } == nil
-      end 
+      end
     end
 
     let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
@@ -601,14 +620,14 @@ describe LogStash::Codecs::CEF do
         insist { e.get("cef_product") } == ""
         insist { e.get("deviceVendor") } == ""
         insist { e.get("deviceProduct") } == ""
-      end 
+      end
     end
 
     let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
     it "should strip leading whitespace from the message" do
       subject.decode(leading_whitespace) do |e|
         validate(e)
-      end 
+      end
     end
 
     let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
@@ -616,7 +635,7 @@ describe LogStash::Codecs::CEF do
       subject.decode(escaped_pipes) do |e|
         ext = e.get('cef_ext')
         insist { ext['moo'] } == 'this\|has an escaped pipe'
-      end 
+      end
     end
 
     let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-cef
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.1.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-02 00:00:00.000000000 Z
+date: 2016-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement