logstash-codec-cef 5.0.2-java → 5.0.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a264129640c226f88fc36cc63ebdd666f37990c8fba3ff5f97e97beb4a7c674
-  data.tar.gz: c34c004be92402ab0d8b768e3a4e41c07c6e48efc8a61f1b8a49ed073f38e7f9
+  metadata.gz: 7c1e2d59b4849c66f6d60d93c0fe03f11e330c97bedfe25280919f3651b5508c
+  data.tar.gz: 4b44ff90abb4bbb14e3a5268df6a841e9354f49ab8fef1c3dfd8ffb6798cde85
 SHA512:
-  metadata.gz: 511c08bfa584988789a5623c91be14f8c9244989dbd84b8eb5bda4043a91862dca29887aaf47991a8857214f59b9c8ba614c22ceaa95375c191a919493404380
-  data.tar.gz: 8000dffe4b20cd9f7e698009337634694a64a3b7674ecbc638c633e64f8f62db80bc7d2b8b6062bc54e1c563ac05305cb52c1703f605864b2831f590b5311eb9
+  metadata.gz: 68f97c0e0361d3b889c62f8502fb2802d24770266e0dc306ee5d327c6b3e9e3405aaf9db9c53e033b46b052bec82b3f8ec9d2df63c99869d5d8e87e1523e1f89
+  data.tar.gz: e2335c058a3d7fbbfa57e57eeb008903b4423063094d543948161864438e8fd65ea09df2a275853991c0ab15122680f8fa4cccb49504ff43fdc9693658d0db75

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.0.3
+ - Fix handling of higher-plane UTF-8 characters in message body
+
 ## 5.0.2
   - Update gemspec summary
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/lib/logstash/codecs/cef.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/util/buftok"
+require "logstash/util/charset"
 require "logstash/codecs/base"
 require "json"
 
@@ -80,6 +81,12 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
   public
   def initialize(params={})
     super(params)
+
+    # CEF input MUST be UTF-8, per the CEF White Paper that serves as the format's specification:
+    # https://web.archive.org/web/20160422182529/https://kc.mcafee.com/resources/sites/MCAFEE/content/live/CORP_KNOWLEDGEBASE/78000/KB78712/en_US/CEF_White_Paper_20100722.pdf
+    @utf8_charset = LogStash::Util::Charset.new('UTF-8')
+    @utf8_charset.logger = self.logger
+
     if @delimiter
       # Logstash configuration doesn't have built-in support for escaping,
       # so we implement it here. Feature discussion for escaping is here:
@@ -110,6 +117,12 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
     event = LogStash::Event.new
     event.set(raw_data_field, data) unless raw_data_field.nil?
 
+    @utf8_charset.convert(data)
+
+    # Several of the many operations in the rest of this method will fail when they encounter UTF8-tagged strings
+    # that contain invalid byte sequences; fail early to avoid wasted work.
+    fail('invalid byte sequence in UTF-8') unless data.valid_encoding?
+
     # Strip any quotations at the start and end, flex connectors seem to send this
     if data[0] == "\""
       data = data[1..-2]

data/logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '5.0.2'
+  s.version         = '5.0.3'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads the ArcSight Common Event Format (CEF)."

data/spec/codecs/cef_spec.rb

@@ -509,11 +509,44 @@ describe LogStash::Codecs::CEF do
       end
     end
 
+    context 'with UTF-8 message' do
+      let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted' }
+
+      # since this spec is encoded UTF-8, the literal strings it contains are encoded with UTF-8,
+      # but codecs in Logstash tend to receive their input as BINARY (or: ASCII-8BIT); ensure that
+      # we can handle either without losing the UTF-8 characters from the higher planes.
+      %w(
+        BINARY
+        UTF-8
+      ).each do |external_encoding|
+        context "externally encoded as #{external_encoding}" do
+          let(:message) { super().force_encoding(external_encoding) }
+          it 'should keep the higher-plane characters' do
+            subject.decode(message.dup) do |event|
+              validate(event)
+              insist { event.get("target") } == "aaaaaああああaaaa"
+              insist { event.get("target").encoding } == Encoding::UTF_8
+            end
+          end
+        end
+      end
+    end
+
+    context 'non-UTF-8 message' do
+      let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
+      it 'should emit message unparsed with _cefparsefailure tag' do
+        subject.decode(message.dup) do |event|
+          insist { event.get("message").bytes.to_a } == message.bytes.to_a
+          insist { event.get("tags") } == ['_cefparsefailure']
+        end
+      end
+    end
+
     context "with raw_data_field set" do
       subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
 
       it "should return the raw message in field message_raw" do
-        subject.decode(message) do |e|
+        subject.decode(message.dup) do |e|
           validate(e)
           insist { e.get("message_raw") } == message
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-cef
 version: !ruby/object:Gem::Version
-  version: 5.0.2
+  version: 5.0.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Reads the ArcSight Common Event Format (CEF).