logstash-codec-cef 6.2.5-java → 6.2.6-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03b13f5507c5f1bdb5f09668a2bcb445dd5d93014b91cb9c17272b7445a787ff
-  data.tar.gz: a25d40bf2ccd77baacc58dd4d49b69da9c7e0b616537d306b937807915d36115
+  metadata.gz: 344660a8caa1f5fbdde48422db80b287e6afff7e8c1d3ebdeb5f70269431a514
+  data.tar.gz: 9f061964eae0cdcd46fcefe9b5feefc05c72074935c44d83a8f35c5e54564a6c
 SHA512:
-  metadata.gz: d600beff671cd1d1a287c153bf4dac27922ea8ca109fc92d3b43c18ea69fdbf645b53067d36cd859604e8013a14010397bd6b0640b4173977ff95a6800f563be
-  data.tar.gz: 4ef42b63bf2c8fdf535c6c7106149dcf7df4e269fd1299708ed3a290d9ceedace58eaf46c002eaa18afb174a2670edf6abea392c120e7eea606de5bb9f8d3bec
+  metadata.gz: 791c750b7085fbefec2e71537d4452174e851bfa28a7d6f046f67d07a543148ebceb51acbc8356b42c90fb2f0fcac28c29ce5f95624ac598070f5389e3f95f72
+  data.tar.gz: c995d8153001929fad98c0dab84664f6af1c6c7b4b873f1be2277d5b0f64e45531178c73fc8fa339ee7c4644c118cf2cfce37c812a4ea6cd6f9d2221d543a470

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 6.2.6
+ - Fix: when decoding, escaped newlines and carriage returns in extension values are now correctly decoded into literal newlines and carriage returns respectively [#98](https://github.com/logstash-plugins/logstash-codec-cef/pull/98)
+ - Fix: when decoding, non-CEF payloads are identified and intercepted to prevent data-loss and corruption. They now cause a descriptive log message to be emitted, and are emitted as their own `_cefparsefailure`-tagged event containing the original bytes in its `message` field [#99](https://github.com/logstash-plugins/logstash-codec-cef/issues/99)
+ - Fix: when decoding while configured with a `delimiter`, flushing this codec now correctly consumes the remainder of its internal buffer. This resolves an issue where bytes that are written without a trailing delimiter could be lost [#100](https://github.com/logstash-plugins/logstash-codec-cef/issues/100) 
+
 ## 6.2.5
   - [DOC] Update link to CEF implementation guide [#97](https://github.com/logstash-plugins/logstash-codec-cef/pull/97)
 

data/lib/logstash/codecs/cef/timestamp_normalizer.rb

@@ -84,9 +84,6 @@ class LogStash::Codecs::CEF::TimestampNormalizer
 
     # Ruby's `Time::at(sec, microseconds_with_frac)`
     Time.at(parsed_time.get_epoch_second, Rational(parsed_time.get_nano, 1000))
-  rescue => e
-    $stderr.puts "parse_cef_format_sgring(#{value.inspect}, #{context_timezone.inspect}) #!=> #{e.message}"
-    raise
   end
 
   def resolve_assuming_year(parsed_temporal_accessor)

data/lib/logstash/codecs/cef.rb

@@ -102,15 +102,12 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   #  - non-pipe characters
   HEADER_PATTERN = /(?:\\\||\\\\|[^|])*?/
 
-  # Cache of a scanner pattern that _captures_ a HEADER followed by an unescaped pipe
-  HEADER_SCANNER = /(#{HEADER_PATTERN})#{Regexp.quote('|')}/
+  # Cache of a scanner pattern that _captures_ a HEADER followed by EOF or an unescaped pipe
+  HEADER_NEXT_FIELD_PATTERN = /(#{HEADER_PATTERN})#{Regexp.quote('|')}/
 
   # Cache of a gsub pattern that matches a backslash-escaped backslash or backslash-escaped pipe, _capturing_ the escaped character
   HEADER_ESCAPE_CAPTURE = /\\([\\|])/
 
-  # Cache of a gsub pattern that matches a backslash-escaped backslash or backslash-escaped equals, _capturing_ the escaped character
-  EXTENSION_VALUE_ESCAPE_CAPTURE = /\\([\\=])/
-
   # While the original CEF spec calls out that extension keys must be alphanumeric and must not contain spaces,
   # in practice many "CEF" producers like the Arcsight smart connector produce non-legal keys including underscores,
   # commas, periods, and square-bracketed index offsets.
@@ -139,8 +136,8 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   # - runs of whitespace that are NOT followed by something that looks like a key-equals sequence
   EXTENSION_VALUE_PATTERN = /(?:\S|\s++(?!#{EXTENSION_KEY_PATTERN}=))*/
 
-  # Cache of a scanner pattern that _captures_ extension field key/value pairs
-  EXTENSION_KEY_VALUE_SCANNER = /(#{EXTENSION_KEY_PATTERN})=(#{EXTENSION_VALUE_PATTERN})\s*/
+  # Cache of a pattern that _captures_ the NEXT extension field key/value pair
+  EXTENSION_NEXT_KEY_VALUE_PATTERN = /^(#{EXTENSION_KEY_PATTERN})=(#{EXTENSION_VALUE_PATTERN})\s*/
 
   ##
   # @see CEF#sanitize_header_field
@@ -160,10 +157,30 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     "="  => "\\=",
     "\n" => "\\n",
     "\r" => "\\n",
-  }
+  }.freeze
   EXTENSION_VALUE_SANITIZER_PATTERN = Regexp.union(EXTENSION_VALUE_SANITIZER_MAPPING.keys)
   private_constant :EXTENSION_VALUE_SANITIZER_MAPPING, :EXTENSION_VALUE_SANITIZER_PATTERN
 
+
+  LITERAL_BACKSLASH = "\\".freeze
+  private_constant :LITERAL_BACKSLASH
+  LITERAL_NEWLINE = "\n".freeze
+  private_constant :LITERAL_NEWLINE
+  LITERAL_CARRIAGE_RETURN = "\r".freeze
+  private_constant :LITERAL_CARRIAGE_RETURN
+
+  ##
+  # @see CEF#desanitize_extension_val
+  EXTENSION_VALUE_SANITIZER_REVERSE_MAPPING = {
+    LITERAL_BACKSLASH+LITERAL_BACKSLASH => LITERAL_BACKSLASH,
+    LITERAL_BACKSLASH+'=' => '=',
+    LITERAL_BACKSLASH+'n' => LITERAL_NEWLINE,
+    LITERAL_BACKSLASH+'r' => LITERAL_CARRIAGE_RETURN,
+  }.freeze
+  EXTENSION_VALUE_SANITIZER_REVERSE_PATTERN = Regexp.union(EXTENSION_VALUE_SANITIZER_REVERSE_MAPPING.keys)
+  private_constant :EXTENSION_VALUE_SANITIZER_REVERSE_MAPPING, :EXTENSION_VALUE_SANITIZER_REVERSE_PATTERN
+
+
   CEF_PREFIX = 'CEF:'.freeze
 
   public
@@ -193,14 +210,24 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   public
   def decode(data, &block)
     if @delimiter
+      @logger.trace("Buffering #{data.bytesize}B of data") if @logger.trace?
       @buffer.extract(data).each do |line|
+        @logger.trace("Decoding #{line.bytesize + @delimiter.bytesize}B of buffered data") if @logger.trace?
         handle(line, &block)
       end
     else
+      @logger.trace("Decoding #{data.bytesize}B of unbuffered data") if @logger.trace?
       handle(data, &block)
     end
   end
 
+  def flush(&block)
+    if @delimiter && (remainder = @buffer.flush)
+      @logger.trace("Flushing #{remainder.bytesize}B of buffered data") if @logger.trace?
+      handle(remainder, &block) unless remainder.empty?
+    end
+  end
+
   def handle(data, &block)
     original_data = data.dup
     event = event_factory.new_event
@@ -218,10 +245,16 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     end
 
     # Use a scanning parser to capture the HEADER_FIELDS
-    unprocessed_data = data
-    @header_fields.each do |field_name|
-      match_data = HEADER_SCANNER.match(unprocessed_data)
-      break if match_data.nil? # missing fields
+    unprocessed_data = data.chomp
+    if unprocessed_data.include?(LITERAL_NEWLINE)
+      fail("message is not valid CEF because it contains unescaped newline characters; " +
+           "use the `delimiter` setting to enable in-codec buffering and delimiter-splitting")
+    end
+    @header_fields.each_with_index do |field_name, idx|
+      match_data = HEADER_NEXT_FIELD_PATTERN.match(unprocessed_data)
+      if match_data.nil?
+        fail("message is not valid CEF; found #{idx} of 7 required pipe-terminated header fields")
+      end
 
       escaped_field_value = match_data[1]
       next if escaped_field_value.nil?
@@ -248,11 +281,14 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     event.set(cef_version_field, delete_cef_prefix(event.get(cef_version_field)))
 
     # Use a scanning parser to capture the Extension Key/Value Pairs
-    if message && message.include?('=')
+    if message && !message.empty?
       message = message.strip
       extension_fields = {}
 
-      message.scan(EXTENSION_KEY_VALUE_SCANNER) do |extension_field_key, raw_extension_field_value|
+      while (match = message.match(EXTENSION_NEXT_KEY_VALUE_PATTERN))
+        extension_field_key, raw_extension_field_value = match.captures
+        message = match.post_match
+
         # expand abbreviated extension field keys
         extension_field_key = @decode_mapping.fetch(extension_field_key, extension_field_key)
 
@@ -260,10 +296,13 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
         extension_field_key = extension_field_key.sub(EXTENSION_KEY_ARRAY_CAPTURE, '[\1]\2') if extension_field_key.end_with?(']')
 
         # process legal extension field value escapes
-        extension_field_value = raw_extension_field_value.gsub(EXTENSION_VALUE_ESCAPE_CAPTURE, '\1')
+        extension_field_value = desanitize_extension_val(raw_extension_field_value)
 
         extension_fields[extension_field_key] = extension_field_value
       end
+      if !message.empty?
+        fail("invalid extensions; keyless value present `#{message}`")
+      end
 
       # in ECS mode, normalize timestamps including timezone.
       if ecs_compatibility != :disabled
@@ -283,7 +322,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     yield event
   rescue => e
     @logger.error("Failed to decode CEF payload. Generating failure event with payload in message field.",
-                  :exception => e.class, :message => e.message, :backtrace => e.backtrace, :original_data => original_data)
+                  log_metadata(:original_data => original_data))
     yield event_factory.new_event("message" => data, "tags" => ["_cefparsefailure"])
   end
 
@@ -332,6 +371,19 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     @syslog_header = ecs_select[disabled:'syslog',v1:'[log][syslog][header]']
   end
 
+  ##
+  # produces log metadata, injecting the current exception and log-level-relevant backtraces
+  # @param context [Hash{Symbol=>Object}]: the base context
+  def log_metadata(context={})
+    return context unless $!
+
+    exception_context = {}
+    exception_context[:exception] = "#{$!.class}: #{$!.message}"
+    exception_context[:backtrace] = $!.backtrace if @logger.debug?
+
+    exception_context.merge(context)
+  end
+
   class CEFField
     ##
     # @param name [String]: the full CEF name of a field
@@ -547,6 +599,10 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
          .gsub(EXTENSION_VALUE_SANITIZER_PATTERN, EXTENSION_VALUE_SANITIZER_MAPPING)
   end
 
+  def desanitize_extension_val(value)
+    value.to_s.gsub(EXTENSION_VALUE_SANITIZER_REVERSE_PATTERN, EXTENSION_VALUE_SANITIZER_REVERSE_MAPPING)
+  end
+
   def normalize_timestamp(value, device_timezone_name)
     value = @timestamp_normalzer.normalize(value, device_timezone_name).iso8601(9)
 

data/logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '6.2.5'
+  s.version         = '6.2.6'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads the ArcSight Common Event Format (CEF)."

data/spec/codecs/cef/timestamp_normalizer_spec.rb

@@ -159,7 +159,6 @@ describe LogStash::Codecs::CEF::TimestampNormalizer do
       context 'and handling a yearless date string from mid january' do
         let(:time_to_parse) { Time.parse("2021-01-17T00:00:08.123456789Z") }
         it 'assumes that the date being parsed is in the distant past' do
-          $stderr.puts(parsable_string)
           expect(parsed_result.month).to eq(1)
           expect(parsed_result.year).to eq(time_of_parse.year)
         end

data/spec/codecs/cef_spec.rb

@@ -409,14 +409,16 @@ describe LogStash::Codecs::CEF do
     # @yieldparam event [Event]
     # @yieldreturn [void]
     # @return [Event]
-    def decode_one(codec, data)
-      events = do_decode(codec, data)
+    def decode_one(codec, data, flush: true, &block)
+      events = do_decode(codec, data, flush: flush)
       fail("Expected one event, got #{events.size} events: #{events.inspect}") unless events.size == 1
       event = events.first
 
-      if block_given?
-        aggregate_failures('decode one') do
-          yield event
+      if block
+        enriched_event_validation(event) do |e|
+          aggregate_failures('decode one') do
+            yield e
+          end
         end
       end
 
@@ -434,16 +436,35 @@ describe LogStash::Codecs::CEF do
     # @yieldparam event [Event]
     # @yieldreturn [void]
     # @return [Array<Event>]
-    def do_decode(codec, data)
+    def do_decode(codec, data, flush: true, &block)
       events = []
       codec.decode(data) do |event|
         events << event
       end
+      flush && codec.flush do |event|
+        events << event
+      end
 
-      events.each { |event| yield event } if block_given?
+      if block
+        events.each do |event|
+          enriched_event_validation(event, &block)
+        end
+      end
 
       events
     end
+
+    ##
+    # Enrich event validation by outputting the serialized event to stderr
+    # if-and-only-if the provided block's rspec expectations are not met.
+    #
+    # @param event [#to_hash_with_metadata]
+    def enriched_event_validation(event)
+      yield(event)
+    rescue RSpec::Expectations::ExpectationNotMetError
+      $stderr.puts("\e[35m#{event.to_hash_with_metadata}\e[0m\n")
+      raise
+    end
   end
 
   context "#decode", :ecs_compatibility_support do
@@ -452,7 +473,7 @@ describe LogStash::Codecs::CEF do
         allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
       end
 
-      let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+      let(:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
 
       include DecodeHelpers
 
@@ -465,17 +486,126 @@ describe LogStash::Codecs::CEF do
         # Related: https://github.com/elastic/logstash/issues/1645
         subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
 
+        let(:message_two) { "CEF:0|fun|whimsy|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+
+        # testing implicit flush when
         it "should parse on the delimiter " do
-          do_decode(subject,message) do |e|
+          do_decode(subject, message, flush: false) do |e|
             raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
           end
 
-          decode_one(subject, "\r\n") do |e|
+          # the delimiter's presence flushes what we already received, but not the new bytes we send
+          decode_one(subject, "\r\n#{message_two}", flush: false) do |e|
+            validate(e)
+            insist { e.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "security"
+            insist { e.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "threatmanager"
+          end
+
+          # allowing a flush emits the buffered event with our new bits appended
+          decode_one(subject, " split=perfect", flush: true) do |e|
+            validate(e)
+            insist { e.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "fun"
+            insist { e.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "whimsy"
+            insist { e.get("split") } == "perfect"
+          end
+        end
+
+        it 'flushes on close' do
+          # message does NOT have delimiter, but we still get our event
+          decode_one(subject, message, flush: true) do |e|
             validate(e)
             insist { e.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "security"
             insist { e.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "threatmanager"
           end
         end
+
+        it 'emits multiple from a single decode operation' do
+          events = do_decode(subject, "#{message}\r\n#{message_two}")
+          expect(events.size).to eq(2)
+
+          enriched_event_validation(events[0]) do |event|
+            validate(event)
+            insist { event.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "security"
+            insist { event.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "threatmanager"
+          end
+
+          enriched_event_validation(events[1]) do |event|
+            validate(event)
+            insist { event.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "fun"
+            insist { event.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "whimsy"
+          end
+        end
+      end
+
+      # CEF requires seven pipe-terminated headers before optional extensions
+      context 'with a non-CEF payload' do
+        let(:logger_stub) { double('Logger').as_null_object }
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:logger).and_return(logger_stub)
+        end
+
+        context 'containing 0 header-like sections' do
+          let(:message) { 'this is not cef' }
+          it 'logs helpfully and produces a tagged event' do
+            do_decode(subject,message) do |event|
+              expect(event.get('tags')).to include('_cefparsefailure')
+              expect(event.get('message')).to eq(message)
+            end
+            expect(logger_stub).to have_received(:error)
+                                     .with(a_string_including('Failed to decode CEF payload. Generating failure event with payload in message field'),
+                                           a_hash_including(exception: a_string_including("found 0 of 7 required pipe-terminated header fields"),
+                                                            original_data: message))
+          end
+        end
+        context 'containing 4 header-like sections' do
+          let(:message) { "a|b|c with several \\| escaped\\| pipes|d|bananas" }
+          it 'logs helpfully and produces a tagged event' do
+            do_decode(subject,message) do |event|
+              expect(event.get('tags')).to include('_cefparsefailure')
+              expect(event.get('message')).to eq(message)
+            end
+            expect(logger_stub).to have_received(:error)
+                                     .with(a_string_including('Failed to decode CEF payload. Generating failure event with payload in message field'),
+                                           a_hash_including(exception: a_string_including("found 4 of 7 required pipe-terminated header fields"),
+                                                            original_data: message))
+          end
+        end
+        context 'containing non-key/value extensions' do
+          let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|this is in the extensions space but it is not valid because it is not equals-separated key/value" }
+          it 'logs helpfully and produces a tagged event' do
+            do_decode(subject,message) do |event|
+              expect(event.get('tags')).to include('_cefparsefailure')
+              expect(event.get('message')).to eq(message)
+            end
+            expect(logger_stub).to have_received(:error)
+                                     .with(a_string_including('Failed to decode CEF payload. Generating failure event with payload in message field'),
+                                           a_hash_including(exception: a_string_including("invalid extensions; keyless value present"),
+                                                            original_data: message))
+          end
+        end
+        context 'containing unescaped newlines' do
+          # when not using a `delimiter`, we expect exactly one CEF log per call to decode.
+          let (:message) {
+            <<~EOMESSAGE
+              CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.67
+              CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.67
+              CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.67
+              CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.67
+              CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.67
+            EOMESSAGE
+          }
+          it 'logs helpfully and produces a tagged event' do
+            do_decode(subject, message) do |event|
+              expect(event.get('tags')).to include('_cefparsefailure')
+              expect(event.get('message')).to eq(message)
+            end
+            expect(logger_stub).to have_received(:error)
+                                     .with(a_string_including('Failed to decode CEF payload. Generating failure event with payload in message field'),
+                                           a_hash_including(exception: a_string_including("message is not valid CEF because it contains unescaped newline characters",
+                                                                                          "use the `delimiter` setting to enable in-codec buffering and delimiter-splitting"),
+                                                            original_data: message))
+          end
+        end
       end
 
       context 'when a CEF header ends with a pair of properly-escaped backslashes' do
@@ -548,6 +678,23 @@ describe LogStash::Codecs::CEF do
         end
       end
 
+      let(:literal_newline)         { "\n" }
+      let(:literal_carriage_return) { "\r" }
+      let(:literal_equals)          { "=" }
+      let(:literal_backslash)       { "\\" }
+      let(:escaped_newline)         { literal_backslash + 'n' }
+      let(:escaped_carriage_return) { literal_backslash + 'r' }
+      let(:escaped_equals)          { literal_backslash + literal_equals }
+      let(:escaped_backslash)       { literal_backslash + literal_backslash }
+      let(:escaped_sequences_in_extension_value) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|foo=bar msg=this message has escaped equals #{escaped_equals} and escaped newlines #{escaped_newline} escaped carriage returns #{escaped_carriage_return} and escaped backslashes #{escaped_backslash} in it bar=baz" }
+      it "decodes embedded newlines, carriage regurns, backslashes, and equals signs" do
+        decode_one(subject, escaped_sequences_in_extension_value) do |e|
+          insist { e.get("foo") } == 'bar'
+          insist { e.get("message") } == "this message has escaped equals #{literal_equals} and escaped newlines #{literal_newline} escaped carriage returns #{literal_carriage_return} and escaped backslashes #{literal_backslash} in it"
+          insist { e.get("bar") } == 'baz'
+        end
+      end
+
       context "zoneless deviceReceiptTime(rt) when deviceTimeZone(dtz) is provided" do
         let(:cef_formatted_timestamp) { 'Jul 19 2017 10:50:21.127' }
         let(:zone_name) { 'Europe/Moscow' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-cef
 version: !ruby/object:Gem::Version
-  version: 6.2.5
+  version: 6.2.6
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-20 00:00:00.000000000 Z
+date: 2022-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement