logstash-codec-cef 6.0.0-java → 6.2.0-java

data/lib/logstash/codecs/cef/timestamp_normalizer.rb

@@ -0,0 +1,115 @@
+# encoding: utf-8
+
+require 'java'
+
+# The CEF specification allows a variety of timestamp formats, some of which
+# cannot be unambiguously parsed to a specific points in time, and may require
+# additional side-channel information to do so, namely:
+#  - the time zone or UTC offset (which MAY be included in a separate field)
+#  - the locale (for parsing abbreviated month names)
+#  - the year (assume "recent")
+#
+# This normalizer attempts to use the provided context and make reasonable
+# assumptions when parsing ambiguous dates.
+class LogStash::Codecs::CEF::TimestampNormalizer
+
+  java_import java.time.Clock
+  java_import java.time.LocalDate
+  java_import java.time.LocalTime
+  java_import java.time.MonthDay
+  java_import java.time.OffsetDateTime
+  java_import java.time.ZoneId
+  java_import java.time.ZonedDateTime
+  java_import java.time.format.DateTimeFormatter
+  java_import java.util.Locale
+
+  def initialize(locale:nil, timezone:nil, clock: Clock.systemUTC)
+    @clock = clock
+
+    java_locale   = locale ? get_locale(locale) : Locale.get_default
+    java_timezone = timezone ? ZoneId.of(timezone) : ZoneId.system_default
+
+    @cef_timestamp_format_parser = DateTimeFormatter
+                                       .ofPattern("MMM dd[ yyyy] HH:mm:ss[.SSSSSSSSS][.SSSSSS][.SSS][ zzz]")
+                                       .withZone(java_timezone)
+                                       .withLocale(java_locale)
+  end
+
+  INTEGER_OR_DECIMAL_PATTERN = /\A[1-9][0-9]*(?:\.[0-9]+)?\z/
+  private_constant :INTEGER_OR_DECIMAL_PATTERN
+
+  # @param value [String,Time,Numeric]
+  #   The value to parse. `Time`s are returned without modification, and `Numeric` values
+  #   are treated as millis-since-epoch (as are fully-numeric strings).
+  #   Strings are parsed unsing any of the supported CEF formats, and when the timestamp
+  #   does not encode a year, we assume the year from contextual information like the
+  #   current time.
+  # @param device_timezone_name [String,nil] (optional):
+  #   If known, the time-zone or UTC offset of the device that encoded the timestamp.
+  #   This value is used to determine the offset when no offset is encoded in the timestamp.
+  #   If not provided, the system default time zone is used instead.
+  # @return [Time]
+  def normalize(value, device_timezone_name=nil)
+    return value if value.kind_of?(Time)
+
+    case value
+    when Numeric                    then Time.at(Rational(value, 1000))
+    when INTEGER_OR_DECIMAL_PATTERN then Time.at(Rational(value, 1000))
+    else
+      parse_cef_format_string(value.to_s, device_timezone_name)
+    end
+  end
+
+  private
+
+  def get_locale(spec)
+    if spec.nil?
+      Locale.get_default
+    elsif spec =~ /\A([a-z]{2})_([A-Z]{2})\z/
+      lang, country = Regexp.last_match(1), Regexp.last_match(2)
+      Locale.new(lang, country)
+    else
+      Locale.for_language_tag(spec)
+    end
+  end
+
+  def parse_cef_format_string(value, context_timezone=nil)
+    cef_timestamp_format_parser = @cef_timestamp_format_parser
+    cef_timestamp_format_parser = cef_timestamp_format_parser.with_zone(java.time.ZoneId.of(context_timezone)) unless context_timezone.nil?
+
+    parsed_time = cef_timestamp_format_parser.parse_best(value,
+                                                         ->(v){ ZonedDateTime.from(v) },
+                                                         ->(v){ OffsetDateTime.from(v) },
+                                                         ->(v){ resolve_assuming_year(v) }).to_instant
+
+    # Ruby's `Time::at(sec, microseconds_with_frac)`
+    Time.at(parsed_time.get_epoch_second, Rational(parsed_time.get_nano, 1000))
+  rescue => e
+    $stderr.puts "parse_cef_format_sgring(#{value.inspect}, #{context_timezone.inspect}) #!=> #{e.message}"
+    raise
+  end
+
+  def resolve_assuming_year(parsed_temporal_accessor)
+    parsed_monthday = MonthDay.from(parsed_temporal_accessor)
+    parsed_time = LocalTime.from(parsed_temporal_accessor)
+    parsed_zone = ZoneId.from(parsed_temporal_accessor)
+
+    now = ZonedDateTime.now(@clock.with_zone(parsed_zone))
+
+    parsed_timestamp_with_current_year = ZonedDateTime.of(parsed_monthday.at_year(now.get_year), parsed_time, parsed_zone)
+
+    if (parsed_timestamp_with_current_year > now.plus_days(2))
+      # e.g., on May 12, parsing a date from May 15 or later is plausibly from
+      # the prior calendar year and not actually from the future
+      return ZonedDateTime.of(parsed_monthday.at_year(now.get_year - 1), parsed_time, parsed_zone)
+    elsif now.get_month_value == 12 && (parsed_timestamp_with_current_year.plus_years(1) <= now.plus_days(2))
+      # e.g., on December 31, parsing a date from January 1 could plausibly be
+      # from the very-near future but next calendar year due to out-of-sync
+      # clocks, mismatched timezones, etc.
+      return ZonedDateTime.of(parsed_monthday.at_year(now.get_year + 1), parsed_time, parsed_zone)
+    else
+      # otherwise, assume current calendar year
+      return parsed_timestamp_with_current_year
+    end
+  end
+end

data/logstash-codec-cef.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-cef'
-  s.version         = '6.0.0'
+  s.version         = '6.2.0'
   s.platform        = 'java'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads the ArcSight Common Event Format (CEF)."
@@ -22,6 +22,8 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'insist'
 end

data/spec/codecs/cef/timestamp_normalizer_spec.rb

@@ -0,0 +1,274 @@
+# encoding: utf-8
+
+require 'logstash/util'
+require "logstash/devutils/rspec/spec_helper"
+require "insist"
+require "logstash/codecs/cef"
+require 'logstash/codecs/cef/timestamp_normalizer'
+
+describe LogStash::Codecs::CEF::TimestampNormalizer do
+
+  subject(:timestamp_normalizer) { described_class.new }
+  let(:parsed_result) { timestamp_normalizer.normalize(parsable_string) }
+
+  context "parsing dates with a year specified" do
+    let(:parsable_string) { "Jun 17 2027 17:57:06.456" }
+    it 'parses the year correctly' do
+      expect(parsed_result.year).to eq(2027)
+    end
+  end
+
+  context "unparsable inputs" do
+    let(:parsable_string) { "Last Thursday" }
+    it "raises a StandardError exception that can be caught upstream" do
+      expect { parsed_result }.to raise_error(StandardError, /#{Regexp::escape parsable_string}/)
+    end
+  end
+
+  context "side-channel time zone indicators" do
+    let(:context_timezone) { 'America/New_York' }
+    let(:parsed_result) { timestamp_normalizer.normalize(parsable_string, context_timezone) }
+
+    context "when parsed input does not include offset information" do
+      let(:parsable_string) { "Jun 17 2027 17:57:06.456" }
+
+      it 'offsets to the context timezone time' do
+        expect(parsed_result).to eq(Time.parse("2027-06-17T21:57:06.456Z"))
+      end
+    end
+    context "when parsed input includes offset information" do
+      let(:parsable_string) { "Jun 17 2027 17:57:06.456 -07:00" }
+
+      it 'uses the parsed offset' do
+        expect(parsed_result).to eq(Time.parse("2027-06-18T00:57:06.456Z"))
+      end
+    end
+    context "when parsed input is a millis-since-epoch timestamp" do
+      let(:parsable_string) { "1616623591694" }
+
+      it "does not offset the time" do
+        expect(parsed_result).to eq(Time.at(Rational(1616623591694,1_000)))
+        expect(parsed_result.nsec).to eq(694_000_000)
+      end
+    end
+    context "when parsed input is a millis-since-epoch timestamp with decimal part and microsecond precision" do
+      let(:parsable_string) { "1616623591694.176" }
+
+      it "does not offset the time" do
+        expect(parsed_result).to eq(Time.at(Rational(1616623591694176,1_000_000)))
+        expect(parsed_result.nsec).to eq(694_176_000)
+      end
+    end
+    context "when parsed input is a millis-since-epoch timestamp with decimal part and nanosecond precision" do
+      let(:parsable_string) { "1616623591694.176789" }
+
+      it "does not offset the time" do
+        expect(parsed_result).to eq(Time.at(Rational(1616623591694176789,1_000_000_000)))
+        expect(parsed_result.nsec).to eq(694_176_789)
+      end
+    end
+  end
+
+  context "when locale is specified" do
+    let(:locale_language) { 'de' }
+    let(:locale_country) { 'DE' }
+    let(:locale_spec) { "#{locale_language}_#{locale_country}" }
+
+    # Due to locale-provider loading changes in JDK 9, abbreviations for months
+    # depend on a combination of the JDK version and the `java.locale.providers`
+    # system property.
+    # Instead of hard-coding a localized month name, use this process's locales
+    # to generate one.
+    let(:java_locale) { java.util.Locale.new(locale_language, locale_country) }
+    let(:localized_march_abbreviation) do
+      months = java.text.DateFormatSymbols.new(java_locale).get_short_months
+      months[2] # march
+    end
+
+    subject(:timestamp_normalizer) { described_class.new(locale: locale_spec) }
+
+    let(:parsable_string) { "#{localized_march_abbreviation} 17 2019 17:57:06.456 +01:00" }
+
+    it 'uses the locale to parse the date' do
+      expect(parsed_result).to eq(Time.parse("2019-03-17T17:57:06.456+01:00"))
+    end
+  end
+
+  context "parsing dates with sub-second precision" do
+    context "whole second precision" do
+      let(:parsable_string) { "Mar 17 2021 12:34:56 +00:00" }
+      it "is accurate to the second" do
+        expect(parsed_result.nsec).to eq(000_000_000)
+        expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56Z"))
+      end
+    end
+    context "millisecond sub-second precision" do
+      let(:parsable_string) { "Mar 17 2021 12:34:56.987" }
+      let(:format_string) { "%b %d %H:%M:%S.%3N" }
+      it "is accurate to the millisecond" do
+        expect(parsed_result.nsec).to eq(987_000_000)
+        expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987Z"))
+      end
+    end
+    context "microsecond sub-second precision" do
+      let(:parsable_string) { "Mar 17 2021 12:34:56.987654" }
+      let(:format_string) { "%b %d %H:%M:%S.%6N" }
+      it "is accurate to the microsecond" do
+        expect(parsed_result.nsec).to eq(987_654_000)
+        expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987654Z"))
+      end
+    end
+    context "nanosecond sub-second precision" do
+      let(:parsable_string) { "Mar 17 2021 12:34:56.987654321" }
+      let(:format_string) { "%b %d %H:%M:%S.%9N" }
+      it "is accurate to the nanosecond" do
+        expect(parsed_result.nsec).to eq(987_654_321)
+        expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987654321Z"))
+      end
+    end
+  end
+
+  context "parsing dates with no year specified" do
+    let(:time_of_parse) { fail(NotImplementedError) }
+    let(:format_to_parse) { "%b %d %H:%M:%S.%3N" }
+    let(:offset_days) { fail(NotImplementedError) }
+    let(:time_to_parse) { (time_of_parse + (offset_days * 86400)) }
+    let(:parsable_string) { time_to_parse.strftime(format_to_parse) }
+
+
+    let(:anchored_clock) do
+      instant = java.time.Instant.of_epoch_second(time_of_parse.to_i)
+      zone = java.time.ZoneId.system_default
+
+      java.time.Clock.fixed(instant, zone)
+    end
+
+    subject(:timestamp_normalizer) { described_class.new(clock: anchored_clock) }
+
+    let(:parsed_result) { timestamp_normalizer.normalize(parsable_string) }
+
+    context 'when parsing a date during late December' do
+      let(:time_of_parse) { Time.parse("2020-12-31T23:53:08.123456789Z") }
+      context 'and handling a date string from early january' do
+        let(:time_to_parse) { Time.parse("2021-01-01T00:00:08.123456789Z") }
+        it 'assumes that the date being parsed is in the very near future' do
+          expect(parsed_result.month).to eq(1)
+          expect(parsed_result.year).to eq(time_of_parse.year + 1)
+        end
+      end
+      context 'and handling a yearless date string from mid january' do
+        let(:time_to_parse) { Time.parse("2021-01-17T00:00:08.123456789Z") }
+        it 'assumes that the date being parsed is in the distant past' do
+          $stderr.puts(parsable_string)
+          expect(parsed_result.month).to eq(1)
+          expect(parsed_result.year).to eq(time_of_parse.year)
+        end
+      end
+    end
+
+    # As a smoke test to validate the guess-the-year feature when the provided CEF timestamp
+    # does not include the year, we iterate through a variety of dates that we want to parse,
+    # and with each of them we parse with a mock clock as if we were performing the parsing
+    # operation at a variety of date-times relative to the timestamp represented.
+    %w(
+      2021-01-20T04:10:22.961Z
+      2021-06-08T03:38:55.518Z
+      2021-07-12T18:46:12.149Z
+      2021-08-12T04:17:36.680Z
+      2021-08-12T13:20:14.951Z
+      2021-09-17T13:18:57.534Z
+      2021-09-23T16:35:40.404Z
+      2021-10-30T18:52:29.263Z
+      2021-11-11T00:52:39.409Z
+      2021-11-19T13:37:07.189Z
+      2021-12-02T01:09:21.846Z
+      2021-12-11T16:35:05.641Z
+      2021-12-15T14:17:22.152Z
+      2021-12-19T05:53:57.200Z
+      2021-12-20T16:18:17.637Z
+      2021-12-22T12:06:48.965Z
+      2021-12-26T04:45:14.964Z
+      2022-01-05T09:42:39.895Z
+      2022-02-02T04:58:22.080Z
+      2022-02-05T08:10:15.386Z
+      2022-02-15T16:48:27.083Z
+      2022-02-31T13:26:55.298Z
+      2022-03-10T20:16:25.732Z
+      2022-03-20T23:38:58.734Z
+      2022-03-30T03:42:09.546Z
+      2022-04-09T05:55:18.697Z
+      2022-04-14T05:05:29.278Z
+      2022-04-25T15:29:19.567Z
+      2022-05-02T08:34:21.666Z
+      2022-05-24T02:59:02.257Z
+      2022-07-25T01:58:35.713Z
+      2022-07-27T03:27:57.568Z
+      2022-07-28T20:28:22.704Z
+      2022-09-21T08:59:10.508Z
+      2022-10-29T23:54:02.372Z
+      2022-11-12T15:22:51.758Z
+      2022-11-22T22:02:33.278Z
+      2022-12-30T03:18:38.333Z
+      2023-01-02T16:55:57.829Z
+      2023-01-13T16:37:38.078Z
+      2023-01-27T07:27:09.296Z
+      2023-01-30T17:56:43.665Z
+      2023-02-18T11:41:18.886Z
+      2023-02-28T18:51:59.504Z
+      2023-03-10T06:52:14.285Z
+      2023-04-17T16:25:06.489Z
+      2023-04-18T20:46:29.611Z
+      2023-04-27T10:21:41.036Z
+      2023-05-08T02:54:57.131Z
+      2023-05-13T01:17:37.396Z
+      2023-05-24T18:23:05.136Z
+      2023-06-01T11:09:48.129Z
+      2023-06-22T07:44:56.876Z
+      2023-06-25T20:17:44.394Z
+      2023-06-25T20:53:36.329Z
+      2023-07-24T13:07:58.536Z
+      2023-07-27T21:35:54.299Z
+      2023-08-07T11:15:33.803Z
+      2023-08-12T18:45:46.791Z
+      2023-08-19T23:22:19.717Z
+      2023-08-22T23:19:41.075Z
+      2023-08-25T15:22:47.405Z
+      2023-09-03T14:34:13.345Z
+      2023-09-28T05:48:20.040Z
+      2023-09-29T21:14:15.531Z
+      2023-11-12T21:25:55.233Z
+      2023-11-30T00:41:21.834Z
+      2023-12-11T10:14:51.676Z
+      2023-12-14T18:02:33.005Z
+      2023-12-18T09:00:43.589Z
+      2023-12-20T20:02:42.205Z
+      2023-12-22T10:13:37.553Z
+      2023-12-27T19:42:37.905Z
+      2023-12-31T17:52:50.101Z
+      2024-02-29T01:23:45.678Z
+    ).map {|ts| Time.parse(ts) }.each do |timestamp|
+      cef_parsable_timestamp = timestamp.strftime("%b %d %H:%M:%S.%3N Z")
+
+      context "when parsing the string `#{cef_parsable_timestamp}`" do
+
+        let(:expected_result) { timestamp }
+        let(:parsable_string) { cef_parsable_timestamp }
+
+        {
+          'very recent past'      =>       -30.789, # ~ 30 seconds ago
+          'somewhat recent past'  =>   -608976.678, # ~ 1 week days ago
+          'distant past'          => -29879991.916, # ~ 11-1/2 months days ago
+          'near future'           =>    132295.719, # ~ 1.5 days from now
+        }.each do |desc, shift|
+          shifted_now = timestamp - shift
+          context "when that string could plausibly be in the #{desc} (NOW: #{shifted_now.iso8601(3)})" do
+            let(:time_of_parse) { shifted_now }
+            it "produces a time in the #{desc} (#{timestamp.iso8601(3)})" do
+              expect(parsed_result).to eq(expected_result)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/codecs/cef_spec.rb

@@ -1,15 +1,19 @@
 # encoding: utf-8
+require 'logstash/util'
 require "logstash/devutils/rspec/spec_helper"
+require "insist"
 require "logstash/codecs/cef"
 require "logstash/event"
 require "json"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+
 describe LogStash::Codecs::CEF do
-  subject do
+  subject(:codec) do
     next LogStash::Codecs::CEF.new
   end
 
-  context "#encode" do
+  context "#encode", :ecs_compatibility_support do
     subject(:codec) { LogStash::Codecs::CEF.new }
 
     let(:results)   { [] }
@@ -209,25 +213,92 @@ describe LogStash::Codecs::CEF do
       codec.encode(event)
       expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|foo=[0-9TZ.:-]+$/m)
     end
-    
-    it "should encode the CEF field names to their long versions" do
-      # This is with the default value of "reverse_mapping" that is "false".
-      codec.on_event{|data, newdata| results << newdata}
-      codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHost", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
-      event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHost" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
-      codec.encode(event)
-      expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=foobar applicationProtocol=foobar deviceCustomIPv6Address1=foobar deviceCustomIPv6Address1Label=foobar deviceCustomIPv6Address2=foobar deviceCustomIPv6Address2Label=foobar deviceCustomIPv6Address3=foobar deviceCustomIPv6Address3Label=foobar deviceCustomIPv6Address4=foobar deviceCustomIPv6Address4Label=foobar deviceEventCategory=foobar deviceCustomFloatingPoint1=foobar deviceCustomFloatingPoint1Label=foobar deviceCustomFloatingPoint2=foobar deviceCustomFloatingPoint2Label=foobar deviceCustomFloatingPoint3=foobar deviceCustomFloatingPoint3Label=foobar deviceCustomFloatingPoint4=foobar deviceCustomFloatingPoint4Label=foobar deviceCustomNumber1=foobar deviceCustomNumber1Label=foobar deviceCustomNumber2=foobar deviceCustomNumber2Label=foobar deviceCustomNumber3=foobar deviceCustomNumber3Label=foobar baseEventCount=foobar deviceCustomString1=foobar deviceCustomString1Label=foobar deviceCustomString2=foobar deviceCustomString2Label=foobar deviceCustomString3=foobar deviceCustomString3Label=foobar deviceCustomString4=foobar deviceCustomString4Label=foobar deviceCustomString5=foobar deviceCustomString5Label=foobar deviceCustomString6=foobar deviceCustomString6Label=foobar destinationHostName=foobar destinationMacAddress=foobar destinationNtDomain=foobar destinationProcessId=foobar destinationUserPrivileges=foobar destinationProcessName=foobar destinationPort=foobar destinationAddress=foobar destinationUserId=foobar destinationUserName=foobar deviceAddress=foobar deviceHostName=foobar deviceProcessId=foobar endTime=foobar fileName=foobar fileSize=foobar bytesIn=foobar message=foobar bytesOut=foobar eventOutcome=foobar transportProtocol=foobar requestUrl=foobar deviceReceiptTime=foobar sourceHostName=foobar sourceMacAddress=foobar sourceNtDomain=foobar sourceProcessId=foobar sourceUserPrivileges=foobar sourceProcessName=foobar sourcePort=foobar sourceAddress=foobar startTime=foobar sourceUserId=foobar sourceUserName=foobar agentHost=foobar agentReceiptTime=foobar agentType=foobar agentId=foobar agentAddress=foobar agentVersion=foobar agentTimeZone=foobar destinationTimeZone=foobar sourceLongitude=foobar sourceLatitude=foobar destinationLongitude=foobar destinationLatitude=foobar categoryDeviceType=foobar managerReceiptTime=foobar agentMacAddress=foobar$/m)
-    end
 
-    context "with reverse_mapping set to true" do
-      subject(:codec) { LogStash::Codecs::CEF.new("reverse_mapping" => true) }
+    ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
 
-      it "should encode the CEF field names to their short versions" do
+      it "should encode the CEF field names to their long versions" do
+        # This is with the default value of "reverse_mapping" that is "false".
         codec.on_event{|data, newdata| results << newdata}
-        codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHost", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
-        event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHost" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
+        codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
+        event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
         codec.encode(event)
-        expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=foobar app=foobar c6a1=foobar c6a1Label=foobar c6a2=foobar c6a2Label=foobar c6a3=foobar c6a3Label=foobar c6a4=foobar c6a4Label=foobar cat=foobar cfp1=foobar cfp1Label=foobar cfp2=foobar cfp2Label=foobar cfp3=foobar cfp3Label=foobar cfp4=foobar cfp4Label=foobar cn1=foobar cn1Label=foobar cn2=foobar cn2Label=foobar cn3=foobar cn3Label=foobar cnt=foobar cs1=foobar cs1Label=foobar cs2=foobar cs2Label=foobar cs3=foobar cs3Label=foobar cs4=foobar cs4Label=foobar cs5=foobar cs5Label=foobar cs6=foobar cs6Label=foobar dhost=foobar dmac=foobar dntdom=foobar dpid=foobar dpriv=foobar dproc=foobar dpt=foobar dst=foobar duid=foobar duser=foobar dvc=foobar dvchost=foobar dvcpid=foobar end=foobar fname=foobar fsize=foobar in=foobar msg=foobar out=foobar outcome=foobar proto=foobar request=foobar rt=foobar shost=foobar smac=foobar sntdom=foobar spid=foobar spriv=foobar sproc=foobar spt=foobar src=foobar start=foobar suid=foobar suser=foobar ahost=foobar art=foobar at=foobar aid=foobar agt=foobar av=foobar atz=foobar dtz=foobar slong=foobar slat=foobar dlong=foobar dlat=foobar catdt=foobar mrt=foobar amac=foobar$/m)
+        expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=foobar applicationProtocol=foobar deviceCustomIPv6Address1=foobar deviceCustomIPv6Address1Label=foobar deviceCustomIPv6Address2=foobar deviceCustomIPv6Address2Label=foobar deviceCustomIPv6Address3=foobar deviceCustomIPv6Address3Label=foobar deviceCustomIPv6Address4=foobar deviceCustomIPv6Address4Label=foobar deviceEventCategory=foobar deviceCustomFloatingPoint1=foobar deviceCustomFloatingPoint1Label=foobar deviceCustomFloatingPoint2=foobar deviceCustomFloatingPoint2Label=foobar deviceCustomFloatingPoint3=foobar deviceCustomFloatingPoint3Label=foobar deviceCustomFloatingPoint4=foobar deviceCustomFloatingPoint4Label=foobar deviceCustomNumber1=foobar deviceCustomNumber1Label=foobar deviceCustomNumber2=foobar deviceCustomNumber2Label=foobar deviceCustomNumber3=foobar deviceCustomNumber3Label=foobar baseEventCount=foobar deviceCustomString1=foobar deviceCustomString1Label=foobar deviceCustomString2=foobar deviceCustomString2Label=foobar deviceCustomString3=foobar deviceCustomString3Label=foobar deviceCustomString4=foobar deviceCustomString4Label=foobar deviceCustomString5=foobar deviceCustomString5Label=foobar deviceCustomString6=foobar deviceCustomString6Label=foobar destinationHostName=foobar destinationMacAddress=foobar destinationNtDomain=foobar destinationProcessId=foobar destinationUserPrivileges=foobar destinationProcessName=foobar destinationPort=foobar destinationAddress=foobar destinationUserId=foobar destinationUserName=foobar deviceAddress=foobar deviceHostName=foobar deviceProcessId=foobar endTime=foobar fileName=foobar fileSize=foobar bytesIn=foobar message=foobar bytesOut=foobar eventOutcome=foobar transportProtocol=foobar requestUrl=foobar deviceReceiptTime=foobar sourceHostName=foobar sourceMacAddress=foobar sourceNtDomain=foobar sourceProcessId=foobar sourceUserPrivileges=foobar sourceProcessName=foobar sourcePort=foobar sourceAddress=foobar startTime=foobar sourceUserId=foobar sourceUserName=foobar agentHostName=foobar agentReceiptTime=foobar agentType=foobar agentId=foobar agentAddress=foobar agentVersion=foobar agentTimeZone=foobar destinationTimeZone=foobar sourceLongitude=foobar sourceLatitude=foobar destinationLongitude=foobar destinationLatitude=foobar categoryDeviceType=foobar managerReceiptTime=foobar agentMacAddress=foobar$/m)
+      end
+
+      if ecs_select.active_mode != :disabled
+        let(:event_flat_hash) do
+          {
+            "[event][action]" => "floop", # deviceAction
+            "[network][protocol]" => "https", # applicationProtocol
+            "[cef][device_custom_ipv6_address_1][value]" => "4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9", # deviceCustomIPv6Address1
+            "[cef][device_custom_ipv6_address_1][label]" => "internal-interface", # deviceCustomIPv6Address1Label
+            "[observer][ip]" => "123.45.67.89", # deviceAddress
+            "[observer][hostname]" => "banana", # deviceHostName
+            "[user_agent][original]" => "'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='", # requestClientApplication
+            "[source][registered_domain]" => "monkey.see" # sourceDnsDomain
+          }
+        end
+
+        let(:event) do
+          event_flat_hash.each_with_object(LogStash::Event.new) do |(fr,v),memo|
+            memo.set(fr, v)
+          end
+        end
+
+        it 'encodes the ECS field names to their CEF name' do
+          codec.on_event{|data, newdata| results << newdata}
+          codec.fields = event_flat_hash.keys
+
+          codec.encode(event)
+
+          expect(results.first).to match(%r{^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=floop applicationProtocol=https deviceCustomIPv6Address1=4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9 deviceCustomIPv6Address1Label=internal-interface deviceAddress=123\.45\.67\.89 deviceHostName=banana requestClientApplication='Foo-Bar/2018\.1\.7; Email:user@example\.com; Guid:test\\=' sourceDnsDomain=monkey.see$}m)
+        end
+      end
+
+      context "with reverse_mapping set to true" do
+        subject(:codec) { LogStash::Codecs::CEF.new("reverse_mapping" => true) }
+
+        it "should encode the CEF field names to their short versions" do
+          codec.on_event{|data, newdata| results << newdata}
+          codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
+          event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
+          codec.encode(event)
+          expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=foobar app=foobar c6a1=foobar c6a1Label=foobar c6a2=foobar c6a2Label=foobar c6a3=foobar c6a3Label=foobar c6a4=foobar c6a4Label=foobar cat=foobar cfp1=foobar cfp1Label=foobar cfp2=foobar cfp2Label=foobar cfp3=foobar cfp3Label=foobar cfp4=foobar cfp4Label=foobar cn1=foobar cn1Label=foobar cn2=foobar cn2Label=foobar cn3=foobar cn3Label=foobar cnt=foobar cs1=foobar cs1Label=foobar cs2=foobar cs2Label=foobar cs3=foobar cs3Label=foobar cs4=foobar cs4Label=foobar cs5=foobar cs5Label=foobar cs6=foobar cs6Label=foobar dhost=foobar dmac=foobar dntdom=foobar dpid=foobar dpriv=foobar dproc=foobar dpt=foobar dst=foobar duid=foobar duser=foobar dvc=foobar dvchost=foobar dvcpid=foobar end=foobar fname=foobar fsize=foobar in=foobar msg=foobar out=foobar outcome=foobar proto=foobar request=foobar rt=foobar shost=foobar smac=foobar sntdom=foobar spid=foobar spriv=foobar sproc=foobar spt=foobar src=foobar start=foobar suid=foobar suser=foobar ahost=foobar art=foobar at=foobar aid=foobar agt=foobar av=foobar atz=foobar dtz=foobar slong=foobar slat=foobar dlong=foobar dlat=foobar catdt=foobar mrt=foobar amac=foobar$/m)
+        end
+
+        if ecs_select.active_mode != :disabled
+          let(:event_flat_hash) do
+            {
+              "[event][action]" => "floop", # act
+              "[network][protocol]" => "https", # app
+              "[cef][device_custom_ipv6_address_1][value]" => "4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9", # c6a1
+              "[cef][device_custom_ipv6_address_1][label]" => "internal-interface", # c6a1Label
+              "[observer][ip]" => "123.45.67.89", # dvc
+              "[observer][hostname]" => "banana", # dvchost
+              "[user_agent][original]" => "'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='",
+              "[source][registered_domain]" => "monkey.see" # sourceDnsDomain
+            }
+          end
+
+          let(:event) do
+            event_flat_hash.each_with_object(LogStash::Event.new) do |(fr,v),memo|
+              memo.set(fr, v)
+            end
+          end
+
+
+          it 'encodes the ECS field names to their CEF keys' do
+            codec.on_event{|data, newdata| results << newdata}
+            codec.fields = event_flat_hash.keys
+
+            codec.encode(event)
+
+            expect(results.first).to match(%r{^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=floop app=https c6a1=4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9 c6a1Label=internal-interface dvc=123\.45\.67\.89 dvchost=banana requestClientApplication='Foo-Bar/2018\.1\.7; Email:user@example\.com; Guid:test\\=' sourceDnsDomain=monkey.see$}m)
+          end
+        end
       end
     end
   end
@@ -305,11 +376,21 @@ describe LogStash::Codecs::CEF do
     end
   end
 
-  context "#decode" do
-    let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
-
+  module DecodeHelpers
     def validate(e)
       insist { e.is_a?(LogStash::Event) }
+      send("validate_ecs_#{ecs_compatibility}", e)
+    end
+
+    def validate_ecs_v1(e)
+      insist { e.get('[cef][version]') } == "0"
+      insist { e.get('[observer][version]') } == "1.0"
+      insist { e.get('[event][code]') } == "100"
+      insist { e.get('[cef][name]') } == "trojan successfully stopped"
+      insist { e.get('[event][severity]') } == "10"
+    end
+
+    def validate_ecs_disabled(e)
       insist { e.get('cefVersion') } == "0"
       insist { e.get('deviceVersion') } == "1.0"
       insist { e.get('deviceEventClassId') } == "100"
@@ -333,7 +414,11 @@ describe LogStash::Codecs::CEF do
       fail("Expected one event, got #{events.size} events: #{events.inspect}") unless events.size == 1
       event = events.first
 
-      yield event if block_given?
+      if block_given?
+        aggregate_failures('decode one') do
+          yield event
+        end
+      end
 
       event
     end
@@ -359,345 +444,456 @@ describe LogStash::Codecs::CEF do
 
       events
     end
+  end
 
-    context "with delimiter set" do
-      # '\r\n' in single quotes to simulate the real input from a config
-      # containing \r\n as 4-character sequence in the config:
-      #
-      #   delimiter => "\r\n"
-      #
-      # Related: https://github.com/elastic/logstash/issues/1645
-      subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
+  context "#decode", :ecs_compatibility_support do
+    ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
 
-      it "should parse on the delimiter " do
-        do_decode(subject,message) do |e|
-          raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
-        end
+      let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
 
-        decode_one(subject, "\r\n") do |e|
-          validate(e)
-          insist { e.get("deviceVendor") } == "security"
-          insist { e.get("deviceProduct") } == "threatmanager"
+      include DecodeHelpers
+
+      context "with delimiter set" do
+        # '\r\n' in single quotes to simulate the real input from a config
+        # containing \r\n as 4-character sequence in the config:
+        #
+        #   delimiter => "\r\n"
+        #
+        # Related: https://github.com/elastic/logstash/issues/1645
+        subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
+
+        it "should parse on the delimiter " do
+          do_decode(subject,message) do |e|
+            raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
+          end
+
+          decode_one(subject, "\r\n") do |e|
+            validate(e)
+            insist { e.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "security"
+            insist { e.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "threatmanager"
+          end
         end
       end
-    end
 
-    context 'when a CEF header ends with a pair of properly-escaped backslashes' do
-      let(:backslash) { '\\' }
-      let(:pipe) { '|' }
-      let(:message) { "CEF:0|security|threatmanager|1.0|100|double backslash" +
-                      backslash + backslash + # escaped backslash
-                      backslash + backslash + # escaped backslash
-                      "|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+      context 'when a CEF header ends with a pair of properly-escaped backslashes' do
+        let(:backslash) { '\\' }
+        let(:pipe) { '|' }
+        let(:message) { "CEF:0|security|threatmanager|1.0|100|double backslash" +
+                        backslash + backslash + # escaped backslash
+                        backslash + backslash + # escaped backslash
+                        "|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
 
-      it 'should include the backslashes unescaped' do
-        event = decode_one(subject, message)
+        it 'should include the backslashes unescaped' do
+          event = decode_one(subject, message)
 
-        expect(event.get('name')).to eq('double backslash' + backslash + backslash )
-        expect(event.get('severity')).to eq('10') # ensure we didn't consume the separator
+          expect(event.get(ecs_select[disabled:'name',    v1:'[cef][name]'])).to eq('double backslash' + backslash + backslash )
+          expect(event.get(ecs_select[disabled:'severity',v1:'[event][severity]'])).to eq('10') # ensure we didn't consume the separator
+        end
       end
-    end
 
-    it "should parse the cef headers" do
-      decode_one(subject, message) do |e|
-        validate(e)
-        insist { e.get("deviceVendor") } == "security"
-        insist { e.get("deviceProduct") } == "threatmanager"
+      it "should parse the cef headers" do
+        decode_one(subject, message) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == "security"
+          insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == "threatmanager"
+        end
       end
-    end
 
-    it "should parse the cef body" do
-      decode_one(subject, message) do |e|
-        insist { e.get("sourceAddress")} == "10.0.0.192"
-        insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("sourcePort") } == "1232"
+      it "should parse the cef body" do
+        decode_one(subject, message) do |e|
+          insist { e.get(ecs_select[disabled:"sourceAddress",     v1:"[source][ip]"])} == "10.0.0.192"
+          insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
+          insist { e.get(ecs_select[disabled:"sourcePort",        v1:"[source][port]"]) } == "1232"
+        end
       end
-    end
 
-    let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
-    it "should be OK with missing CEF headers (multiple pipes in sequence)" do
-      decode_one(subject, missing_headers) do |e|
-        validate(e)
-        insist { e.get("deviceVendor") } == ""
-        insist { e.get("deviceProduct") } == ""
+      let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+      it "should be OK with missing CEF headers (multiple pipes in sequence)" do
+        decode_one(subject, missing_headers) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == ""
+          insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == ""
+        end
       end
-    end
 
-    let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
-    it "should strip leading whitespace from the message" do
-      decode_one(subject, leading_whitespace) do |e|
-        validate(e)
+      let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+      it "should strip leading whitespace from the message" do
+        decode_one(subject, leading_whitespace) do |e|
+          validate(e)
+        end
       end
-    end
 
-    let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
-    it "should be OK with escaped pipes in the message" do
-      decode_one(subject, escaped_pipes) do |e|
-        insist { e.get("moo") } == 'this\|has an escaped pipe'
+      let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
+      it "should be OK with escaped pipes in the message" do
+        decode_one(subject, escaped_pipes) do |e|
+          insist { e.get("moo") } == 'this\|has an escaped pipe'
+        end
       end
-    end
 
-    let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
-    it "should be OK with not escaped pipes in the message" do
-      decode_one(subject, pipes_in_message) do |e|
-        insist { e.get("moo") } == 'this|has an pipe'
+      let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
+      it "should be OK with not escaped pipes in the message" do
+        decode_one(subject, pipes_in_message) do |e|
+          insist { e.get("moo") } == 'this|has an pipe'
+        end
       end
-    end
 
-    # while we may see these in practice, equals MUST be escaped in the extensions per the spec.
-    let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
-    it "should be OK with equal in the message" do
-      decode_one(subject, equal_in_message) do |e|
-        insist { e.get("moo") } == 'this =has = equals='
+      # while we may see these in practice, equals MUST be escaped in the extensions per the spec.
+      let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
+      it "should be OK with equal in the message" do
+        decode_one(subject, equal_in_message) do |e|
+          insist { e.get("moo") } == 'this =has = equals='
+        end
       end
-    end
 
-    let(:malformed_unescaped_equals_in_extension_value) { %q{CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar} }
-    it 'should split correctly' do
-      decode_one(subject, malformed_unescaped_equals_in_extension_value) do |event|
-        expect(event.get('cefVersion')).to eq('0')
-        expect(event.get('deviceVendor')).to eq('FooBar')
-        expect(event.get('deviceProduct')).to eq('Web Gateway')
-        expect(event.get('deviceVersion')).to eq('1.2.3.45.67')
-        expect(event.get('deviceEventClassId')).to eq('200')
-        expect(event.get('name')).to eq('Success')
-        expect(event.get('severity')).to eq('2')
-
-        # extension key/value pairs
-        expect(event.get('deviceReceiptTime')).to eq('Sep 07 2018 14:50:39')
-        expect(event.get('deviceEventCategory')).to eq('Access Log')
-        expect(event.get('deviceVersion')).to eq('1.2.3.45.67')
-        expect(event.get('destinationAddress')).to eq('1.1.1.1')
-        expect(event.get('destinationHostName')).to eq('foo.example.com')
-        expect(event.get('sourceUserName')).to eq('redacted')
-        expect(event.get('sourceAddress')).to eq('2.2.2.2')
-        expect(event.get('requestMethod')).to eq('POST')
-        expect(event.get('requestUrl')).to eq(%q{'https://foo.example.com/bar/bingo/1'})
-        # Although the value for `requestClientApplication` contains an illegal unquoted equals sign, the sequence
-        # preceeding the unescaped-equals isn't shaped like a key, so we allow it to be a part of the value.
-        expect(event.get('requestClientApplication')).to eq(%q{'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='})
-        expect(event.get('deviceCustomString1Label')).to eq('Foo Bar')
-        expect(event.get('deviceCustomString1')).to eq('')
+      context "zoneless deviceReceiptTime(rt) when deviceTimeZone(dtz) is provided" do
+        let(:cef_formatted_timestamp) { 'Jul 19 2017 10:50:21.127' }
+        let(:zone_name) { 'Europe/Moscow' }
+
+        let(:utc_timestamp) { Time.iso8601("2017-07-19T07:50:21.127Z") } # In summer of 2017, Europe/Moscow was UTC+03:00
+
+        let(:destination_time_zoned) { %Q{CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|Very-High| eventId=1 msg=Worm successfully stopped art=1500464384997 deviceSeverity=10 rt=#{cef_formatted_timestamp} src=10.0.0.1 sourceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 10.0.0.0-10.255.255.255 spt=1232 dst=2.1.2.2 destinationZoneURI=/All Zones/ArcSight System/Public Address Space Zones/RIPE NCC/2.0.0.0-2.255.255.255 (RIPE NCC) ahost=connector.rhel72 agt=192.168.231.129 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-51-8A-84 av=7.6.0.8009.0 atz=Europe/Lisbon at=syslog_file dvchost=client1 dtz=#{zone_name} _cefVer=0.1 aid=3UBajWl0BABCABBzZSlmUdw==} }
+
+        if ecs_select.active_mode == :disabled
+          it 'persists deviceReceiptTime and deviceTimeZone verbatim' do
+            decode_one(subject, destination_time_zoned) do |event|
+              expect(event.get('deviceReceiptTime')).to eq("Jul 19 2017 10:50:21.127")
+              expect(event.get('deviceTimeZone')).to eq('Europe/Moscow')
+            end
+          end
+        else
+          it 'sets the @timestamp using the value in `rt` combined with the offset provided by `dtz`' do
+            decode_one(subject, destination_time_zoned) do |event|
+              expected_time = LogStash::Timestamp.new(utc_timestamp)
+              expect(event.get('[@timestamp]').to_s).to eq(expected_time.to_s)
+              expect(event.get('[event][timezone]')).to eq(zone_name)
+            end
+          end
+        end
+      end
+
+      let(:malformed_unescaped_equals_in_extension_value) { %q{CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar} }
+      it 'should split correctly' do
+        decode_one(subject, malformed_unescaped_equals_in_extension_value) do |event|
+          expect(event.get(ecs_select[disabled:"cefVersion",        v1:"[cef][version]"])).to eq('0')
+          expect(event.get(ecs_select[disabled:"deviceVendor",      v1:"[observer][vendor]"])).to eq('FooBar')
+          expect(event.get(ecs_select[disabled:"deviceProduct",     v1:"[observer][product]"])).to eq('Web Gateway')
+          expect(event.get(ecs_select[disabled:"deviceVersion",     v1:"[observer][version]"])).to eq('1.2.3.45.67')
+          expect(event.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"])).to eq('200')
+          expect(event.get(ecs_select[disabled:"name",              v1:"[cef][name]"])).to eq('Success')
+          expect(event.get(ecs_select[disabled:"severity",          v1:"[event][severity]"])).to eq('2')
+
+          # extension key/value pairs
+          if ecs_compatibility == :disabled
+            expect(event.get('deviceReceiptTime')).to eq('Sep 07 2018 14:50:39')
+          else
+            expected_time = LogStash::Timestamp.new(Time.parse('Sep 07 2018 14:50:39')).to_s
+            expect(event.get('[@timestamp]').to_s).to eq(expected_time)
+          end
+          expect(event.get(ecs_select[disabled:'deviceEventCategory',     v1:'[cef][category]'])).to eq('Access Log')
+          expect(event.get(ecs_select[disabled:'deviceVersion',           v1:'[observer][version]'])).to eq('1.2.3.45.67')
+          expect(event.get(ecs_select[disabled:'destinationAddress',      v1:'[destination][ip]'])).to eq('1.1.1.1')
+          expect(event.get(ecs_select[disabled:'destinationHostName',     v1:'[destination][domain]'])).to eq('foo.example.com')
+          expect(event.get(ecs_select[disabled:'sourceUserName',          v1:'[source][user][name]'])).to eq('redacted')
+          expect(event.get(ecs_select[disabled:'sourceAddress',           v1:'[source][ip]'])).to eq('2.2.2.2')
+          expect(event.get(ecs_select[disabled:'requestMethod',           v1:'[http][request][method]'])).to eq('POST')
+          expect(event.get(ecs_select[disabled:'requestUrl',              v1:'[url][original]'])).to eq(%q{'https://foo.example.com/bar/bingo/1'})
+          # Although the value for `requestClientApplication` contains an illegal unquoted equals sign, the sequence
+          # preceeding the unescaped-equals isn't shaped like a key, so we allow it to be a part of the value.
+          expect(event.get(ecs_select[disabled:'requestClientApplication',v1:'[user_agent][original]'])).to eq(%q{'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='})
+          expect(event.get(ecs_select[disabled:'deviceCustomString1Label',v1:'[cef][device_custom_string_1][label]'])).to eq('Foo Bar')
+          expect(event.get(ecs_select[disabled:'deviceCustomString1',     v1:'[cef][device_custom_string_1][value]'])).to eq('')
+        end
       end
-    end
 
-    context('escaped-equals and unescaped-spaces in the extension values') do
-      let(:query_string) { 'key1=value1&key2=value3 aa.bc&key3=value4'}
-      let(:escaped_query_string) { query_string.gsub('=','\\=') }
-      let(:cef_message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|go=start now query_string=#{escaped_query_string} final=done" }
+      context('escaped-equals and unescaped-spaces in the extension values') do
+        let(:query_string) { 'key1=value1&key2=value3 aa.bc&key3=value4'}
+        let(:escaped_query_string) { query_string.gsub('=','\\=') }
+        let(:cef_message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|go=start now query_string=#{escaped_query_string} final=done" }
 
-      it 'captures the extension values correctly' do
-        event = decode_one(subject, cef_message)
+        it 'captures the extension values correctly' do
+          event = decode_one(subject, cef_message)
 
-        expect(event.get('go')).to eq('start now')
-        expect(event.get('query_string')).to eq(query_string)
-        expect(event.get('final')).to eq('done')
+          expect(event.get('go')).to eq('start now')
+          expect(event.get('query_string')).to eq(query_string)
+          expect(event.get('final')).to eq('done')
+        end
       end
-    end
 
-    let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
-    it "should be OK with escaped backslash in the headers" do
-      decode_one(subject, escaped_backslash_in_header) do |e|
-        insist { e.get("cefVersion") } == '0'
-        insist { e.get("deviceVendor") } == 'secu\\rity'
-        insist { e.get("deviceProduct") } == 'threat\\manager'
-        insist { e.get("deviceVersion") } == '1.\\0'
-        insist { e.get("deviceEventClassId") } == '10\\0'
-        insist { e.get("name") } == 'tro\\jan successfully stopped'
-        insist { e.get("severity") } == '\\10'
+      let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
+      it "should be OK with escaped backslash in the headers" do
+        decode_one(subject, escaped_backslash_in_header) do |e|
+          insist { e.get(ecs_select[disabled:"cefVersion",        v1:"[cef][version]"]) } == '0'
+          insist { e.get(ecs_select[disabled:"deviceVendor",      v1:"[observer][vendor]"]) } == 'secu\\rity'
+          insist { e.get(ecs_select[disabled:"deviceProduct",     v1:"[observer][product]"]) } == 'threat\\manager'
+          insist { e.get(ecs_select[disabled:"deviceVersion",     v1:"[observer][version]"]) } == '1.\\0'
+          insist { e.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"]) } == '10\\0'
+          insist { e.get(ecs_select[disabled:"name",              v1:"[cef][name]"]) } == 'tro\\jan successfully stopped'
+          insist { e.get(ecs_select[disabled:"severity",          v1:"[event][severity]"]) } == '\\10'
+        end
       end
-    end
 
-    let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
-    it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
-      decode_one(subject, escaped_backslash_in_header_edge_case) do |e|
-        validate(e)
-        insist { e.get("deviceVendor") } == 'security\\|'
-        insist { e.get("deviceProduct") } == 'threatmanager\\'
+      let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
+      it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
+        decode_one(subject, escaped_backslash_in_header_edge_case) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'security\\|'
+          insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == 'threatmanager\\'
+        end
       end
-    end
 
-    let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
-    it "should be OK with escaped pipes in the headers" do
-      decode_one(subject, escaped_pipes_in_header) do |e|
-        insist { e.get("cefVersion") } == '0'
-        insist { e.get("deviceVendor") } == 'secu|rity'
-        insist { e.get("deviceProduct") } == 'threatmanager|'
-        insist { e.get("deviceVersion") } == '1.|0'
-        insist { e.get("deviceEventClassId") } == '10|0'
-        insist { e.get("name") } == 'tro|jan successfully stopped'
-        insist { e.get("severity") } == '|10'
+      let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
+      it "should be OK with escaped pipes in the headers" do
+        decode_one(subject, escaped_pipes_in_header) do |e|
+          insist { e.get(ecs_select[disabled:"cefVersion",        v1:"[cef][version]"]) } == '0'
+          insist { e.get(ecs_select[disabled:"deviceVendor",      v1:"[observer][vendor]"]) } == 'secu|rity'
+          insist { e.get(ecs_select[disabled:"deviceProduct",     v1:"[observer][product]"]) } == 'threatmanager|'
+          insist { e.get(ecs_select[disabled:"deviceVersion",     v1:"[observer][version]"]) } == '1.|0'
+          insist { e.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"]) } == '10|0'
+          insist { e.get(ecs_select[disabled:"name",              v1:"[cef][name]"]) } == 'tro|jan successfully stopped'
+          insist { e.get(ecs_select[disabled:"severity",          v1:"[event][severity]"]) } == '|10'
+        end
       end
-    end
 
-    let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
-    it "should be OK with backslashs in the message" do
-      decode_one(subject, backslash_in_message) do |e|
-        insist { e.get("moo") } == 'this \\has \\ backslashs\\'
+      let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
+      it "should be OK with backslashs in the message" do
+        decode_one(subject, backslash_in_message) do |e|
+          insist { e.get("moo") } == 'this \\has \\ backslashs\\'
+        end
       end
-    end
 
-    let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
-    it "should be OK with equal in the headers" do
-      decode_one(subject, equal_in_header) do |e|
-        validate(e)
-        insist { e.get("deviceProduct") } == "threatmanager=equal"
+      let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
+      it "should be OK with equal in the headers" do
+        decode_one(subject, equal_in_header) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == "threatmanager=equal"
+        end
       end
-    end
 
-    let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192  dst=12.121.122.82  spt=1232'}
-    it "should be OK to have one or more spaces between keys" do
-      decode_one(subject, spaces_in_between_keys) do |e|
-        validate(e)
-        insist { e.get("sourceAddress") } == "10.0.0.192"
-        insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("sourcePort") } == "1232"
+      let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192  dst=12.121.122.82  spt=1232'}
+      it "should be OK to have one or more spaces between keys" do
+        decode_one(subject, spaces_in_between_keys) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
+          insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
+          insist { e.get(ecs_select[disabled:"sourcePort",v1:"[source][port]"]) } == "1232"
+        end
       end
-    end
 
-    let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82  spt=1232 dproc=InternetExplorer x.x.x.x'}
-    it "should be OK to have one or more spaces in values" do
-      decode_one(subject, allow_spaces_in_values) do |e|
-        validate(e)
-        insist { e.get("sourceAddress") } == "10.0.0.192"
-        insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("sourcePort") } == "1232"
-        insist { e.get("destinationProcessName") } == "InternetExplorer x.x.x.x"
+      let (:dots_in_keys) {'CEF:0|Vendor|Device|Version|13|my message|5|dvchost=loghost cat=traffic deviceSeverity=notice ad.nn=TEST src=192.168.0.1 destinationPort=53'}
+      it "should be OK with dots in keys" do
+        decode_one(subject, dots_in_keys) do |e|
+          insist { e.get(ecs_select[disabled:"deviceHostName",v1:"[observer][hostname]"]) } == "loghost"
+          insist { e.get("ad.nn") } == 'TEST'
+          insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == '192.168.0.1'
+          insist { e.get(ecs_select[disabled:"destinationPort",v1:"[destination][port]"]) } == '53'
+        end
       end
-    end
 
-    let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
-    it "should keep ad.fields" do
-      decode_one(subject, preserve_additional_fields_with_dot_notations) do |e|
-        validate(e)
-        insist { e.get("sourceAddress") } == "10.0.0.192"
-        insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("[ad.field][0]") } == "field0"
-        insist { e.get("[ad.name][1]") } == "new_name"
-        insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
-        insist { e.get('ad.Error_,Code') } == "3221225578"
-        insist { e.get("additional.dotfieldName") } == "new_value"
+      let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82  spt=1232 dproc=InternetExplorer x.x.x.x'}
+      it "should be OK to have one or more spaces in values" do
+        decode_one(subject, allow_spaces_in_values) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
+          insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
+          insist { e.get(ecs_select[disabled:"sourcePort",v1:"[source][port]"]) } == "1232"
+          insist { e.get(ecs_select[disabled:"destinationProcessName",v1:"[destination][process][name]"]) } == "InternetExplorer x.x.x.x"
+        end
       end
-    end
 
-    let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000  23041810181C0000  /CN\=random.user/OU\=User Login End-Entity  /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
-    it "should correctly parse random values even with additional fields in message" do
-      decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
-        validate(e)
-        insist { e.get("sourceAddress") } == "10.0.0.192"
-        insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("[ad.field][0]") } == "field0"
-        insist { e.get("[ad.name][1]") } == "new_name"
-        insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
-        insist { e.get("ad.Error_,Code") } == "3221225578"
-        insist { e.get("additional.dotfieldName") } == "new_value"
-        insist { e.get("deviceCustomString4") } == "401 random.user Admin 0 23041A10181C0000  23041810181C0000  /CN\=random.user/OU\=User Login End-Entity  /CN\=TEST/OU\=Login CA TEST 34"
+      let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
+      it "should keep ad.fields" do
+        decode_one(subject, preserve_additional_fields_with_dot_notations) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
+          insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
+          insist { e.get("[ad.field][0]") } == "field0"
+          insist { e.get("[ad.name][1]") } == "new_name"
+          insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
+          insist { e.get('ad.Error_,Code') } == "3221225578"
+          insist { e.get("additional.dotfieldName") } == "new_value"
+        end
       end
-    end
 
-    let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
-    it "should preserve unmatched key mappings" do
-      decode_one(subject, preserve_unmatched_key_mappings) do |e|
-        validate(e)
-        insist { e.get("sourceAddress") } == "10.0.0.192"
-        insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("new_key_by_device") } == "new_values here"
+      let(:preserve_complex_multiple_dot_notation_in_extension_fields) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name' }
+      it "should keep ad.fields" do
+        decode_one(subject, preserve_complex_multiple_dot_notation_in_extension_fields) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
+          insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
+          insist { e.get("[ad.field][0]") } == "field0"
+          insist { e.get("[ad.foo.name][1]") } == "new_name"
+          insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
+          insist { e.get('ad.Error_,Code') } == "3221225578"
+          insist { e.get("additional.dotfieldName") } == "new_value"
+        end
       end
-    end
 
-    let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
-    it "should translate most known abbreviated CEF field names" do
-      decode_one(subject, translate_abbreviated_cef_fields) do |e|
-        validate(e)
-        insist { e.get("sourceAddress") } == "10.0.0.192"
-        insist { e.get("destinationAddress") } == "12.121.122.82"
-        insist { e.get("transportProtocol") } == "TCP"
-        insist { e.get("sourceHostName") } == "source.host.name"
-        insist { e.get("destinationHostName") } == "destination.host.name"
-        insist { e.get("sourcePort") } == "11024"
-        insist { e.get("destinationPort") } == "9200"
-        insist { e.get("eventOutcome") } == "Success"
-        insist { e.get("agentMacAddress")} == "00:80:48:1c:24:91"
+      let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000  23041810181C0000  /CN\=random.user/OU\=User Login End-Entity  /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
+      it "should correctly parse random values even with additional fields in message" do
+        decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
+          insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
+          insist { e.get("[ad.field][0]") } == "field0"
+          insist { e.get("[ad.name][1]") } == "new_name"
+          insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
+          insist { e.get("ad.Error_,Code") } == "3221225578"
+          insist { e.get("additional.dotfieldName") } == "new_value"
+          insist { e.get(ecs_select[disabled:"deviceCustomString4",v1:"[cef][device_custom_string_4][value]"]) } == "401 random.user Admin 0 23041A10181C0000  23041810181C0000  /CN\=random.user/OU\=User Login End-Entity  /CN\=TEST/OU\=Login CA TEST 34"
+        end
+      end
+
+      let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
+      it "should preserve unmatched key mappings" do
+        decode_one(subject, preserve_unmatched_key_mappings) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
+          insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
+          insist { e.get("new_key_by_device") } == "new_values here"
+        end
       end
-    end
 
-    let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
-    it "Should detect headers before CEF starts" do
-      decode_one(subject, syslog) do |e|
-        validate(e)
-        insist { e.get('syslog') } == 'Syslogdate Sysloghost'
+      let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
+      it "should translate most known abbreviated CEF field names" do
+        decode_one(subject, translate_abbreviated_cef_fields) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:"sourceAddress",      v1:"[source][ip]"]) } == "10.0.0.192"
+          insist { e.get(ecs_select[disabled:"destinationAddress", v1:"[destination][ip]"]) } == "12.121.122.82"
+          insist { e.get(ecs_select[disabled:"transportProtocol",  v1:"[network][transport]"]) } == "TCP"
+          insist { e.get(ecs_select[disabled:"sourceHostName",     v1:"[source][domain]"]) } == "source.host.name"
+          insist { e.get(ecs_select[disabled:"destinationHostName",v1:"[destination][domain]"]) } == "destination.host.name"
+          insist { e.get(ecs_select[disabled:"sourcePort",         v1:"[source][port]"]) } == "11024"
+          insist { e.get(ecs_select[disabled:"destinationPort",    v1:"[destination][port]"]) } == "9200"
+          insist { e.get(ecs_select[disabled:"eventOutcome",       v1:"[event][outcome]"]) } == "Success"
+          insist { e.get(ecs_select[disabled:"agentMacAddress",    v1:"[agent][mac]"])} == "00:80:48:1c:24:91"
+        end
       end
-    end
 
-    context 'with UTF-8 message' do
-      let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted' }
-
-      # since this spec is encoded UTF-8, the literal strings it contains are encoded with UTF-8,
-      # but codecs in Logstash tend to receive their input as BINARY (or: ASCII-8BIT); ensure that
-      # we can handle either without losing the UTF-8 characters from the higher planes.
-      %w(
-        BINARY
-        UTF-8
-      ).each do |external_encoding|
-        context "externally encoded as #{external_encoding}" do
-          let(:message) { super().force_encoding(external_encoding) }
-          it 'should keep the higher-plane characters' do
-            decode_one(subject, message.dup) do |event|
-              validate(event)
-              insist { event.get("target") } == "aaaaaああああaaaa"
-              insist { event.get("target").encoding } == Encoding::UTF_8
+      let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
+      it "Should detect headers before CEF starts" do
+        decode_one(subject, syslog) do |e|
+          validate(e)
+          insist { e.get(ecs_select[disabled:'syslog',v1:'[log][syslog][header]']) } == 'Syslogdate Sysloghost'
+        end
+      end
+
+      context 'with UTF-8 message' do
+        let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted' }
+
+        # since this spec is encoded UTF-8, the literal strings it contains are encoded with UTF-8,
+        # but codecs in Logstash tend to receive their input as BINARY (or: ASCII-8BIT); ensure that
+        # we can handle either without losing the UTF-8 characters from the higher planes.
+        %w(
+          BINARY
+          UTF-8
+        ).each do |external_encoding|
+          context "externally encoded as #{external_encoding}" do
+            let(:message) { super().force_encoding(external_encoding) }
+            it 'should keep the higher-plane characters' do
+              decode_one(subject, message.dup) do |event|
+                validate(event)
+                insist { event.get("target") } == "aaaaaああああaaaa"
+                insist { event.get("target").encoding } == Encoding::UTF_8
+              end
             end
           end
         end
       end
-    end
 
-    context 'non-UTF-8 message' do
-      let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
-      it 'should emit message unparsed with _cefparsefailure tag' do
-        decode_one(subject, message.dup) do |event|
-          insist { event.get("message").bytes.to_a } == message.bytes.to_a
-          insist { event.get("tags") } == ['_cefparsefailure']
+      context 'non-UTF-8 message' do
+        let(:logger_stub) { double('Logger').as_null_object }
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:logger).and_return(logger_stub)
+        end
+        let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
+        it 'should emit message unparsed with _cefparsefailure tag' do
+          decode_one(subject, message.dup) do |event|
+            insist { event.get("message").bytes.to_a } == message.bytes.to_a
+            insist { event.get("tags") } == ['_cefparsefailure']
+          end
+          expect(logger_stub).to have_received(:error).with(/Failed to decode CEF payload/, any_args)
         end
       end
-    end
 
-    context "with raw_data_field set" do
-      subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
+      context "with raw_data_field set" do
+        subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
 
-      it "should return the raw message in field message_raw" do
-        decode_one(subject, message.dup) do |e|
-          validate(e)
-          insist { e.get("message_raw") } == message
+        it "should return the raw message in field message_raw" do
+          decode_one(subject, message.dup) do |e|
+            validate(e)
+            insist { e.get("message_raw") } == message
+          end
+        end
+      end
+
+      context "legacy aliases" do
+        let(:cef_line) { "CEF:0|security|threatmanager|1.0|100|target acquired|10|destinationLongitude=-73.614830 destinationLatitude=45.505918 sourceLongitude=45.4628328 sourceLatitude=9.1076927" }
+
+        it ecs_select[disabled:"creates the fields as provided",v1:"maps to ECS fields"] do
+          decode_one(codec, cef_line.dup) do |event|
+            #                           |---- LEGACY: AS-PROVIDED ----| |--------- ECS: MAP TO FIELD ----------|
+            expect(event.get(ecs_select[disabled:'destinationLongitude',v1:'[destination][geo][location][lon]'])).to eq('-73.614830')
+            expect(event.get(ecs_select[disabled:'destinationLatitude', v1:'[destination][geo][location][lat]'])).to eq('45.505918')
+            expect(event.get(ecs_select[disabled:'sourceLongitude',     v1:'[source][geo][location][lon]'     ])).to eq('45.4628328')
+            expect(event.get(ecs_select[disabled:'sourceLatitude',      v1:'[source][geo][location][lat]'     ])).to eq('9.1076927')
+          end
         end
       end
     end
   end
 
-  context "encode and decode" do
+  context "encode and decode", :ecs_compatibility_support do
     subject(:codec) { LogStash::Codecs::CEF.new }
 
     let(:results)   { [] }
 
-    it "should return an equal event if encoded and decoded again" do
-      codec.on_event{|data, newdata| results << newdata}
-      codec.vendor = "%{deviceVendor}"
-      codec.product = "%{deviceProduct}"
-      codec.version = "%{deviceVersion}"
-      codec.signature = "%{deviceEventClassId}"
-      codec.name = "%{name}"
-      codec.severity = "%{severity}"
-      codec.fields = [ "foo" ]
-      event = LogStash::Event.new("deviceVendor" => "vendor", "deviceProduct" => "product", "deviceVersion" => "2.0", "deviceEventClassId" => "signature", "name" => "name", "severity" => "1", "foo" => "bar")
-      codec.encode(event)
-      codec.decode(results.first) do |e|
-        expect(e.get('deviceVendor')).to be == event.get('deviceVendor')
-        expect(e.get('deviceProduct')).to be == event.get('deviceProduct')
-        expect(e.get('deviceVersion')).to be == event.get('deviceVersion')
-        expect(e.get('deviceEventClassId')).to be == event.get('deviceEventClassId')
-        expect(e.get('name')).to be == event.get('name')
-        expect(e.get('severity')).to be == event.get('severity')
-        expect(e.get('foo')).to be == event.get('foo')
+    ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
+
+      let(:vendor_field)    { ecs_select[disabled:'deviceVendor',       v1:'[observer][vendor]'] }
+      let(:product_field)   { ecs_select[disabled:'deviceProduct',      v1:'[observer][product]']}
+      let(:version_field)   { ecs_select[disabled:'deviceVersion',      v1:'[observer][version]']}
+      let(:signature_field) { ecs_select[disabled:'deviceEventClassId', v1:'[event][code]']}
+      let(:name_field)      { ecs_select[disabled:'name',               v1:'[cef][name]']}
+      let(:severity_field)  { ecs_select[disabled:'severity',           v1:'[event][severity]']}
+
+      let(:source_dns_domain_field) { ecs_select[disabled:'sourceDnsDomain',v1:'[source][registered_domain]'] }
+
+      it "should return an equal event if encoded and decoded again" do
+        codec.on_event{|data, newdata| results << newdata}
+        codec.vendor = "%{" + vendor_field + "}"
+        codec.product = "%{" + product_field + "}"
+        codec.version = "%{" + version_field + "}"
+        codec.signature = "%{" + signature_field + "}"
+        codec.name = "%{" + name_field + "}"
+        codec.severity = "%{" + severity_field + "}"
+        codec.fields = [ "foo", source_dns_domain_field ]
+        event = LogStash::Event.new.tap do |e|
+          e.set(vendor_field, "vendor")
+          e.set(product_field, "product")
+          e.set(version_field, "2.0")
+          e.set(signature_field, "signature")
+          e.set(name_field, "name")
+          e.set(severity_field, "1")
+          e.set("foo", "bar")
+          e.set(source_dns_domain_field, "apple")
+        end
+        codec.encode(event)
+        codec.decode(results.first) do |e|
+          expect(e.get(vendor_field)).to be == event.get(vendor_field)
+          expect(e.get(product_field)).to be == event.get(product_field)
+          expect(e.get(version_field)).to be == event.get(version_field)
+          expect(e.get(signature_field)).to be == event.get(signature_field)
+          expect(e.get(name_field)).to be == event.get(name_field)
+          expect(e.get(severity_field)).to be == event.get(severity_field)
+          expect(e.get('foo')).to be == event.get('foo')
+          expect(e.get(source_dns_domain_field)).to be == event.get(source_dns_domain_field)
+        end
       end
     end
   end