logstash-codec-cef 6.0.1-java → 6.2.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -0
  3. data/LICENSE +199 -10
  4. data/README.md +1 -1
  5. data/docs/index.asciidoc +368 -9
  6. data/lib/logstash/codecs/cef.rb +342 -151
  7. data/lib/logstash/codecs/cef/timestamp_normalizer.rb +115 -0
  8. data/logstash-codec-cef.gemspec +3 -1
  9. data/spec/codecs/cef/timestamp_normalizer_spec.rb +274 -0
  10. data/spec/codecs/cef_spec.rb +480 -308
  11. metadata +33 -2
data/lib/logstash/codecs/cef/timestamp_normalizer.rb ADDED
@@ -0,0 +1,115 @@
1
+ # encoding: utf-8
2
+
3
+ require 'java'
4
+
5
+ # The CEF specification allows a variety of timestamp formats, some of which
6
+ # cannot be unambiguously parsed to a specific points in time, and may require
7
+ # additional side-channel information to do so, namely:
8
+ # - the time zone or UTC offset (which MAY be included in a separate field)
9
+ # - the locale (for parsing abbreviated month names)
10
+ # - the year (assume "recent")
11
+ #
12
+ # This normalizer attempts to use the provided context and make reasonable
13
+ # assumptions when parsing ambiguous dates.
14
+ class LogStash::Codecs::CEF::TimestampNormalizer
15
+
16
+ java_import java.time.Clock
17
+ java_import java.time.LocalDate
18
+ java_import java.time.LocalTime
19
+ java_import java.time.MonthDay
20
+ java_import java.time.OffsetDateTime
21
+ java_import java.time.ZoneId
22
+ java_import java.time.ZonedDateTime
23
+ java_import java.time.format.DateTimeFormatter
24
+ java_import java.util.Locale
25
+
26
+ def initialize(locale:nil, timezone:nil, clock: Clock.systemUTC)
27
+ @clock = clock
28
+
29
+ java_locale = locale ? get_locale(locale) : Locale.get_default
30
+ java_timezone = timezone ? ZoneId.of(timezone) : ZoneId.system_default
31
+
32
+ @cef_timestamp_format_parser = DateTimeFormatter
33
+ .ofPattern("MMM dd[ yyyy] HH:mm:ss[.SSSSSSSSS][.SSSSSS][.SSS][ zzz]")
34
+ .withZone(java_timezone)
35
+ .withLocale(java_locale)
36
+ end
37
+
38
+ INTEGER_OR_DECIMAL_PATTERN = /\A[1-9][0-9]*(?:\.[0-9]+)?\z/
39
+ private_constant :INTEGER_OR_DECIMAL_PATTERN
40
+
41
+ # @param value [String,Time,Numeric]
42
+ # The value to parse. `Time`s are returned without modification, and `Numeric` values
43
+ # are treated as millis-since-epoch (as are fully-numeric strings).
44
+ # Strings are parsed unsing any of the supported CEF formats, and when the timestamp
45
+ # does not encode a year, we assume the year from contextual information like the
46
+ # current time.
47
+ # @param device_timezone_name [String,nil] (optional):
48
+ # If known, the time-zone or UTC offset of the device that encoded the timestamp.
49
+ # This value is used to determine the offset when no offset is encoded in the timestamp.
50
+ # If not provided, the system default time zone is used instead.
51
+ # @return [Time]
52
+ def normalize(value, device_timezone_name=nil)
53
+ return value if value.kind_of?(Time)
54
+
55
+ case value
56
+ when Numeric then Time.at(Rational(value, 1000))
57
+ when INTEGER_OR_DECIMAL_PATTERN then Time.at(Rational(value, 1000))
58
+ else
59
+ parse_cef_format_string(value.to_s, device_timezone_name)
60
+ end
61
+ end
62
+
63
+ private
64
+
65
+ def get_locale(spec)
66
+ if spec.nil?
67
+ Locale.get_default
68
+ elsif spec =~ /\A([a-z]{2})_([A-Z]{2})\z/
69
+ lang, country = Regexp.last_match(1), Regexp.last_match(2)
70
+ Locale.new(lang, country)
71
+ else
72
+ Locale.for_language_tag(spec)
73
+ end
74
+ end
75
+
76
+ def parse_cef_format_string(value, context_timezone=nil)
77
+ cef_timestamp_format_parser = @cef_timestamp_format_parser
78
+ cef_timestamp_format_parser = cef_timestamp_format_parser.with_zone(java.time.ZoneId.of(context_timezone)) unless context_timezone.nil?
79
+
80
+ parsed_time = cef_timestamp_format_parser.parse_best(value,
81
+ ->(v){ ZonedDateTime.from(v) },
82
+ ->(v){ OffsetDateTime.from(v) },
83
+ ->(v){ resolve_assuming_year(v) }).to_instant
84
+
85
+ # Ruby's `Time::at(sec, microseconds_with_frac)`
86
+ Time.at(parsed_time.get_epoch_second, Rational(parsed_time.get_nano, 1000))
87
+ rescue => e
88
+ $stderr.puts "parse_cef_format_sgring(#{value.inspect}, #{context_timezone.inspect}) #!=> #{e.message}"
89
+ raise
90
+ end
91
+
92
+ def resolve_assuming_year(parsed_temporal_accessor)
93
+ parsed_monthday = MonthDay.from(parsed_temporal_accessor)
94
+ parsed_time = LocalTime.from(parsed_temporal_accessor)
95
+ parsed_zone = ZoneId.from(parsed_temporal_accessor)
96
+
97
+ now = ZonedDateTime.now(@clock.with_zone(parsed_zone))
98
+
99
+ parsed_timestamp_with_current_year = ZonedDateTime.of(parsed_monthday.at_year(now.get_year), parsed_time, parsed_zone)
100
+
101
+ if (parsed_timestamp_with_current_year > now.plus_days(2))
102
+ # e.g., on May 12, parsing a date from May 15 or later is plausibly from
103
+ # the prior calendar year and not actually from the future
104
+ return ZonedDateTime.of(parsed_monthday.at_year(now.get_year - 1), parsed_time, parsed_zone)
105
+ elsif now.get_month_value == 12 && (parsed_timestamp_with_current_year.plus_years(1) <= now.plus_days(2))
106
+ # e.g., on December 31, parsing a date from January 1 could plausibly be
107
+ # from the very-near future but next calendar year due to out-of-sync
108
+ # clocks, mismatched timezones, etc.
109
+ return ZonedDateTime.of(parsed_monthday.at_year(now.get_year + 1), parsed_time, parsed_zone)
110
+ else
111
+ # otherwise, assume current calendar year
112
+ return parsed_timestamp_with_current_year
113
+ end
114
+ end
115
+ end
data/logstash-codec-cef.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-cef'
4
- s.version = '6.0.1'
4
+ s.version = '6.2.1'
5
5
  s.platform = 'java'
6
6
  s.licenses = ['Apache License (2.0)']
7
7
  s.summary = "Reads the ArcSight Common Event Format (CEF)."
@@ -22,6 +22,8 @@ Gem::Specification.new do |s|
22
22
 
23
23
  # Gem dependencies
24
24
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
25
+ s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
25
26
 
26
27
  s.add_development_dependency 'logstash-devutils'
28
+ s.add_development_dependency 'insist'
27
29
  end
data/spec/codecs/cef/timestamp_normalizer_spec.rb ADDED
@@ -0,0 +1,274 @@
1
+ # encoding: utf-8
2
+
3
+ require 'logstash/util'
4
+ require "logstash/devutils/rspec/spec_helper"
5
+ require "insist"
6
+ require "logstash/codecs/cef"
7
+ require 'logstash/codecs/cef/timestamp_normalizer'
8
+
9
+ describe LogStash::Codecs::CEF::TimestampNormalizer do
10
+
11
+ subject(:timestamp_normalizer) { described_class.new }
12
+ let(:parsed_result) { timestamp_normalizer.normalize(parsable_string) }
13
+
14
+ context "parsing dates with a year specified" do
15
+ let(:parsable_string) { "Jun 17 2027 17:57:06.456" }
16
+ it 'parses the year correctly' do
17
+ expect(parsed_result.year).to eq(2027)
18
+ end
19
+ end
20
+
21
+ context "unparsable inputs" do
22
+ let(:parsable_string) { "Last Thursday" }
23
+ it "raises a StandardError exception that can be caught upstream" do
24
+ expect { parsed_result }.to raise_error(StandardError, /#{Regexp::escape parsable_string}/)
25
+ end
26
+ end
27
+
28
+ context "side-channel time zone indicators" do
29
+ let(:context_timezone) { 'America/New_York' }
30
+ let(:parsed_result) { timestamp_normalizer.normalize(parsable_string, context_timezone) }
31
+
32
+ context "when parsed input does not include offset information" do
33
+ let(:parsable_string) { "Jun 17 2027 17:57:06.456" }
34
+
35
+ it 'offsets to the context timezone time' do
36
+ expect(parsed_result).to eq(Time.parse("2027-06-17T21:57:06.456Z"))
37
+ end
38
+ end
39
+ context "when parsed input includes offset information" do
40
+ let(:parsable_string) { "Jun 17 2027 17:57:06.456 -07:00" }
41
+
42
+ it 'uses the parsed offset' do
43
+ expect(parsed_result).to eq(Time.parse("2027-06-18T00:57:06.456Z"))
44
+ end
45
+ end
46
+ context "when parsed input is a millis-since-epoch timestamp" do
47
+ let(:parsable_string) { "1616623591694" }
48
+
49
+ it "does not offset the time" do
50
+ expect(parsed_result).to eq(Time.at(Rational(1616623591694,1_000)))
51
+ expect(parsed_result.nsec).to eq(694_000_000)
52
+ end
53
+ end
54
+ context "when parsed input is a millis-since-epoch timestamp with decimal part and microsecond precision" do
55
+ let(:parsable_string) { "1616623591694.176" }
56
+
57
+ it "does not offset the time" do
58
+ expect(parsed_result).to eq(Time.at(Rational(1616623591694176,1_000_000)))
59
+ expect(parsed_result.nsec).to eq(694_176_000)
60
+ end
61
+ end
62
+ context "when parsed input is a millis-since-epoch timestamp with decimal part and nanosecond precision" do
63
+ let(:parsable_string) { "1616623591694.176789" }
64
+
65
+ it "does not offset the time" do
66
+ expect(parsed_result).to eq(Time.at(Rational(1616623591694176789,1_000_000_000)))
67
+ expect(parsed_result.nsec).to eq(694_176_789)
68
+ end
69
+ end
70
+ end
71
+
72
+ context "when locale is specified" do
73
+ let(:locale_language) { 'de' }
74
+ let(:locale_country) { 'DE' }
75
+ let(:locale_spec) { "#{locale_language}_#{locale_country}" }
76
+
77
+ # Due to locale-provider loading changes in JDK 9, abbreviations for months
78
+ # depend on a combination of the JDK version and the `java.locale.providers`
79
+ # system property.
80
+ # Instead of hard-coding a localized month name, use this process's locales
81
+ # to generate one.
82
+ let(:java_locale) { java.util.Locale.new(locale_language, locale_country) }
83
+ let(:localized_march_abbreviation) do
84
+ months = java.text.DateFormatSymbols.new(java_locale).get_short_months
85
+ months[2] # march
86
+ end
87
+
88
+ subject(:timestamp_normalizer) { described_class.new(locale: locale_spec) }
89
+
90
+ let(:parsable_string) { "#{localized_march_abbreviation} 17 2019 17:57:06.456 +01:00" }
91
+
92
+ it 'uses the locale to parse the date' do
93
+ expect(parsed_result).to eq(Time.parse("2019-03-17T17:57:06.456+01:00"))
94
+ end
95
+ end
96
+
97
+ context "parsing dates with sub-second precision" do
98
+ context "whole second precision" do
99
+ let(:parsable_string) { "Mar 17 2021 12:34:56 +00:00" }
100
+ it "is accurate to the second" do
101
+ expect(parsed_result.nsec).to eq(000_000_000)
102
+ expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56Z"))
103
+ end
104
+ end
105
+ context "millisecond sub-second precision" do
106
+ let(:parsable_string) { "Mar 17 2021 12:34:56.987" }
107
+ let(:format_string) { "%b %d %H:%M:%S.%3N" }
108
+ it "is accurate to the millisecond" do
109
+ expect(parsed_result.nsec).to eq(987_000_000)
110
+ expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987Z"))
111
+ end
112
+ end
113
+ context "microsecond sub-second precision" do
114
+ let(:parsable_string) { "Mar 17 2021 12:34:56.987654" }
115
+ let(:format_string) { "%b %d %H:%M:%S.%6N" }
116
+ it "is accurate to the microsecond" do
117
+ expect(parsed_result.nsec).to eq(987_654_000)
118
+ expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987654Z"))
119
+ end
120
+ end
121
+ context "nanosecond sub-second precision" do
122
+ let(:parsable_string) { "Mar 17 2021 12:34:56.987654321" }
123
+ let(:format_string) { "%b %d %H:%M:%S.%9N" }
124
+ it "is accurate to the nanosecond" do
125
+ expect(parsed_result.nsec).to eq(987_654_321)
126
+ expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987654321Z"))
127
+ end
128
+ end
129
+ end
130
+
131
+ context "parsing dates with no year specified" do
132
+ let(:time_of_parse) { fail(NotImplementedError) }
133
+ let(:format_to_parse) { "%b %d %H:%M:%S.%3N" }
134
+ let(:offset_days) { fail(NotImplementedError) }
135
+ let(:time_to_parse) { (time_of_parse + (offset_days * 86400)) }
136
+ let(:parsable_string) { time_to_parse.strftime(format_to_parse) }
137
+
138
+
139
+ let(:anchored_clock) do
140
+ instant = java.time.Instant.of_epoch_second(time_of_parse.to_i)
141
+ zone = java.time.ZoneId.system_default
142
+
143
+ java.time.Clock.fixed(instant, zone)
144
+ end
145
+
146
+ subject(:timestamp_normalizer) { described_class.new(clock: anchored_clock) }
147
+
148
+ let(:parsed_result) { timestamp_normalizer.normalize(parsable_string) }
149
+
150
+ context 'when parsing a date during late December' do
151
+ let(:time_of_parse) { Time.parse("2020-12-31T23:53:08.123456789Z") }
152
+ context 'and handling a date string from early january' do
153
+ let(:time_to_parse) { Time.parse("2021-01-01T00:00:08.123456789Z") }
154
+ it 'assumes that the date being parsed is in the very near future' do
155
+ expect(parsed_result.month).to eq(1)
156
+ expect(parsed_result.year).to eq(time_of_parse.year + 1)
157
+ end
158
+ end
159
+ context 'and handling a yearless date string from mid january' do
160
+ let(:time_to_parse) { Time.parse("2021-01-17T00:00:08.123456789Z") }
161
+ it 'assumes that the date being parsed is in the distant past' do
162
+ $stderr.puts(parsable_string)
163
+ expect(parsed_result.month).to eq(1)
164
+ expect(parsed_result.year).to eq(time_of_parse.year)
165
+ end
166
+ end
167
+ end
168
+
169
+ # As a smoke test to validate the guess-the-year feature when the provided CEF timestamp
170
+ # does not include the year, we iterate through a variety of dates that we want to parse,
171
+ # and with each of them we parse with a mock clock as if we were performing the parsing
172
+ # operation at a variety of date-times relative to the timestamp represented.
173
+ %w(
174
+ 2021-01-20T04:10:22.961Z
175
+ 2021-06-08T03:38:55.518Z
176
+ 2021-07-12T18:46:12.149Z
177
+ 2021-08-12T04:17:36.680Z
178
+ 2021-08-12T13:20:14.951Z
179
+ 2021-09-17T13:18:57.534Z
180
+ 2021-09-23T16:35:40.404Z
181
+ 2021-10-30T18:52:29.263Z
182
+ 2021-11-11T00:52:39.409Z
183
+ 2021-11-19T13:37:07.189Z
184
+ 2021-12-02T01:09:21.846Z
185
+ 2021-12-11T16:35:05.641Z
186
+ 2021-12-15T14:17:22.152Z
187
+ 2021-12-19T05:53:57.200Z
188
+ 2021-12-20T16:18:17.637Z
189
+ 2021-12-22T12:06:48.965Z
190
+ 2021-12-26T04:45:14.964Z
191
+ 2022-01-05T09:42:39.895Z
192
+ 2022-02-02T04:58:22.080Z
193
+ 2022-02-05T08:10:15.386Z
194
+ 2022-02-15T16:48:27.083Z
195
+ 2022-02-31T13:26:55.298Z
196
+ 2022-03-10T20:16:25.732Z
197
+ 2022-03-20T23:38:58.734Z
198
+ 2022-03-30T03:42:09.546Z
199
+ 2022-04-09T05:55:18.697Z
200
+ 2022-04-14T05:05:29.278Z
201
+ 2022-04-25T15:29:19.567Z
202
+ 2022-05-02T08:34:21.666Z
203
+ 2022-05-24T02:59:02.257Z
204
+ 2022-07-25T01:58:35.713Z
205
+ 2022-07-27T03:27:57.568Z
206
+ 2022-07-28T20:28:22.704Z
207
+ 2022-09-21T08:59:10.508Z
208
+ 2022-10-29T23:54:02.372Z
209
+ 2022-11-12T15:22:51.758Z
210
+ 2022-11-22T22:02:33.278Z
211
+ 2022-12-30T03:18:38.333Z
212
+ 2023-01-02T16:55:57.829Z
213
+ 2023-01-13T16:37:38.078Z
214
+ 2023-01-27T07:27:09.296Z
215
+ 2023-01-30T17:56:43.665Z
216
+ 2023-02-18T11:41:18.886Z
217
+ 2023-02-28T18:51:59.504Z
218
+ 2023-03-10T06:52:14.285Z
219
+ 2023-04-17T16:25:06.489Z
220
+ 2023-04-18T20:46:29.611Z
221
+ 2023-04-27T10:21:41.036Z
222
+ 2023-05-08T02:54:57.131Z
223
+ 2023-05-13T01:17:37.396Z
224
+ 2023-05-24T18:23:05.136Z
225
+ 2023-06-01T11:09:48.129Z
226
+ 2023-06-22T07:44:56.876Z
227
+ 2023-06-25T20:17:44.394Z
228
+ 2023-06-25T20:53:36.329Z
229
+ 2023-07-24T13:07:58.536Z
230
+ 2023-07-27T21:35:54.299Z
231
+ 2023-08-07T11:15:33.803Z
232
+ 2023-08-12T18:45:46.791Z
233
+ 2023-08-19T23:22:19.717Z
234
+ 2023-08-22T23:19:41.075Z
235
+ 2023-08-25T15:22:47.405Z
236
+ 2023-09-03T14:34:13.345Z
237
+ 2023-09-28T05:48:20.040Z
238
+ 2023-09-29T21:14:15.531Z
239
+ 2023-11-12T21:25:55.233Z
240
+ 2023-11-30T00:41:21.834Z
241
+ 2023-12-11T10:14:51.676Z
242
+ 2023-12-14T18:02:33.005Z
243
+ 2023-12-18T09:00:43.589Z
244
+ 2023-12-20T20:02:42.205Z
245
+ 2023-12-22T10:13:37.553Z
246
+ 2023-12-27T19:42:37.905Z
247
+ 2023-12-31T17:52:50.101Z
248
+ 2024-02-29T01:23:45.678Z
249
+ ).map {|ts| Time.parse(ts) }.each do |timestamp|
250
+ cef_parsable_timestamp = timestamp.strftime("%b %d %H:%M:%S.%3N Z")
251
+
252
+ context "when parsing the string `#{cef_parsable_timestamp}`" do
253
+
254
+ let(:expected_result) { timestamp }
255
+ let(:parsable_string) { cef_parsable_timestamp }
256
+
257
+ {
258
+ 'very recent past' => -30.789, # ~ 30 seconds ago
259
+ 'somewhat recent past' => -608976.678, # ~ 1 week days ago
260
+ 'distant past' => -29879991.916, # ~ 11-1/2 months days ago
261
+ 'near future' => 132295.719, # ~ 1.5 days from now
262
+ }.each do |desc, shift|
263
+ shifted_now = timestamp - shift
264
+ context "when that string could plausibly be in the #{desc} (NOW: #{shifted_now.iso8601(3)})" do
265
+ let(:time_of_parse) { shifted_now }
266
+ it "produces a time in the #{desc} (#{timestamp.iso8601(3)})" do
267
+ expect(parsed_result).to eq(expected_result)
268
+ end
269
+ end
270
+ end
271
+ end
272
+ end
273
+ end
274
+ end
data/spec/codecs/cef_spec.rb CHANGED
@@ -1,15 +1,19 @@
1
1
  # encoding: utf-8
2
+ require 'logstash/util'
2
3
  require "logstash/devutils/rspec/spec_helper"
4
+ require "insist"
3
5
  require "logstash/codecs/cef"
4
6
  require "logstash/event"
5
7
  require "json"
6
8
 
9
+ require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
10
+
7
11
  describe LogStash::Codecs::CEF do
8
- subject do
12
+ subject(:codec) do
9
13
  next LogStash::Codecs::CEF.new
10
14
  end
11
15
 
12
- context "#encode" do
16
+ context "#encode", :ecs_compatibility_support do
13
17
  subject(:codec) { LogStash::Codecs::CEF.new }
14
18
 
15
19
  let(:results) { [] }
@@ -209,25 +213,92 @@ describe LogStash::Codecs::CEF do
209
213
  codec.encode(event)
210
214
  expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|foo=[0-9TZ.:-]+$/m)
211
215
  end
212
-
213
- it "should encode the CEF field names to their long versions" do
214
- # This is with the default value of "reverse_mapping" that is "false".
215
- codec.on_event{|data, newdata| results << newdata}
216
- codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHost", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
217
- event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHost" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
218
- codec.encode(event)
219
- expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=foobar applicationProtocol=foobar deviceCustomIPv6Address1=foobar deviceCustomIPv6Address1Label=foobar deviceCustomIPv6Address2=foobar deviceCustomIPv6Address2Label=foobar deviceCustomIPv6Address3=foobar deviceCustomIPv6Address3Label=foobar deviceCustomIPv6Address4=foobar deviceCustomIPv6Address4Label=foobar deviceEventCategory=foobar deviceCustomFloatingPoint1=foobar deviceCustomFloatingPoint1Label=foobar deviceCustomFloatingPoint2=foobar deviceCustomFloatingPoint2Label=foobar deviceCustomFloatingPoint3=foobar deviceCustomFloatingPoint3Label=foobar deviceCustomFloatingPoint4=foobar deviceCustomFloatingPoint4Label=foobar deviceCustomNumber1=foobar deviceCustomNumber1Label=foobar deviceCustomNumber2=foobar deviceCustomNumber2Label=foobar deviceCustomNumber3=foobar deviceCustomNumber3Label=foobar baseEventCount=foobar deviceCustomString1=foobar deviceCustomString1Label=foobar deviceCustomString2=foobar deviceCustomString2Label=foobar deviceCustomString3=foobar deviceCustomString3Label=foobar deviceCustomString4=foobar deviceCustomString4Label=foobar deviceCustomString5=foobar deviceCustomString5Label=foobar deviceCustomString6=foobar deviceCustomString6Label=foobar destinationHostName=foobar destinationMacAddress=foobar destinationNtDomain=foobar destinationProcessId=foobar destinationUserPrivileges=foobar destinationProcessName=foobar destinationPort=foobar destinationAddress=foobar destinationUserId=foobar destinationUserName=foobar deviceAddress=foobar deviceHostName=foobar deviceProcessId=foobar endTime=foobar fileName=foobar fileSize=foobar bytesIn=foobar message=foobar bytesOut=foobar eventOutcome=foobar transportProtocol=foobar requestUrl=foobar deviceReceiptTime=foobar sourceHostName=foobar sourceMacAddress=foobar sourceNtDomain=foobar sourceProcessId=foobar sourceUserPrivileges=foobar sourceProcessName=foobar sourcePort=foobar sourceAddress=foobar startTime=foobar sourceUserId=foobar sourceUserName=foobar agentHost=foobar agentReceiptTime=foobar agentType=foobar agentId=foobar agentAddress=foobar agentVersion=foobar agentTimeZone=foobar destinationTimeZone=foobar sourceLongitude=foobar sourceLatitude=foobar destinationLongitude=foobar destinationLatitude=foobar categoryDeviceType=foobar managerReceiptTime=foobar agentMacAddress=foobar$/m)
220
- end
221
216
 
222
- context "with reverse_mapping set to true" do
223
- subject(:codec) { LogStash::Codecs::CEF.new("reverse_mapping" => true) }
217
+ ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
218
+ before(:each) do
219
+ allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
220
+ end
224
221
 
225
- it "should encode the CEF field names to their short versions" do
222
+ it "should encode the CEF field names to their long versions" do
223
+ # This is with the default value of "reverse_mapping" that is "false".
226
224
  codec.on_event{|data, newdata| results << newdata}
227
- codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHost", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
228
- event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHost" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
225
+ codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
226
+ event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
229
227
  codec.encode(event)
230
- expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=foobar app=foobar c6a1=foobar c6a1Label=foobar c6a2=foobar c6a2Label=foobar c6a3=foobar c6a3Label=foobar c6a4=foobar c6a4Label=foobar cat=foobar cfp1=foobar cfp1Label=foobar cfp2=foobar cfp2Label=foobar cfp3=foobar cfp3Label=foobar cfp4=foobar cfp4Label=foobar cn1=foobar cn1Label=foobar cn2=foobar cn2Label=foobar cn3=foobar cn3Label=foobar cnt=foobar cs1=foobar cs1Label=foobar cs2=foobar cs2Label=foobar cs3=foobar cs3Label=foobar cs4=foobar cs4Label=foobar cs5=foobar cs5Label=foobar cs6=foobar cs6Label=foobar dhost=foobar dmac=foobar dntdom=foobar dpid=foobar dpriv=foobar dproc=foobar dpt=foobar dst=foobar duid=foobar duser=foobar dvc=foobar dvchost=foobar dvcpid=foobar end=foobar fname=foobar fsize=foobar in=foobar msg=foobar out=foobar outcome=foobar proto=foobar request=foobar rt=foobar shost=foobar smac=foobar sntdom=foobar spid=foobar spriv=foobar sproc=foobar spt=foobar src=foobar start=foobar suid=foobar suser=foobar ahost=foobar art=foobar at=foobar aid=foobar agt=foobar av=foobar atz=foobar dtz=foobar slong=foobar slat=foobar dlong=foobar dlat=foobar catdt=foobar mrt=foobar amac=foobar$/m)
228
+ expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=foobar applicationProtocol=foobar deviceCustomIPv6Address1=foobar deviceCustomIPv6Address1Label=foobar deviceCustomIPv6Address2=foobar deviceCustomIPv6Address2Label=foobar deviceCustomIPv6Address3=foobar deviceCustomIPv6Address3Label=foobar deviceCustomIPv6Address4=foobar deviceCustomIPv6Address4Label=foobar deviceEventCategory=foobar deviceCustomFloatingPoint1=foobar deviceCustomFloatingPoint1Label=foobar deviceCustomFloatingPoint2=foobar deviceCustomFloatingPoint2Label=foobar deviceCustomFloatingPoint3=foobar deviceCustomFloatingPoint3Label=foobar deviceCustomFloatingPoint4=foobar deviceCustomFloatingPoint4Label=foobar deviceCustomNumber1=foobar deviceCustomNumber1Label=foobar deviceCustomNumber2=foobar deviceCustomNumber2Label=foobar deviceCustomNumber3=foobar deviceCustomNumber3Label=foobar baseEventCount=foobar deviceCustomString1=foobar deviceCustomString1Label=foobar deviceCustomString2=foobar deviceCustomString2Label=foobar deviceCustomString3=foobar deviceCustomString3Label=foobar deviceCustomString4=foobar deviceCustomString4Label=foobar deviceCustomString5=foobar deviceCustomString5Label=foobar deviceCustomString6=foobar deviceCustomString6Label=foobar destinationHostName=foobar destinationMacAddress=foobar destinationNtDomain=foobar destinationProcessId=foobar destinationUserPrivileges=foobar destinationProcessName=foobar destinationPort=foobar destinationAddress=foobar destinationUserId=foobar destinationUserName=foobar deviceAddress=foobar deviceHostName=foobar deviceProcessId=foobar endTime=foobar fileName=foobar fileSize=foobar bytesIn=foobar message=foobar bytesOut=foobar eventOutcome=foobar transportProtocol=foobar requestUrl=foobar deviceReceiptTime=foobar sourceHostName=foobar sourceMacAddress=foobar sourceNtDomain=foobar sourceProcessId=foobar sourceUserPrivileges=foobar sourceProcessName=foobar sourcePort=foobar sourceAddress=foobar startTime=foobar sourceUserId=foobar sourceUserName=foobar agentHostName=foobar agentReceiptTime=foobar agentType=foobar agentId=foobar agentAddress=foobar agentVersion=foobar agentTimeZone=foobar destinationTimeZone=foobar sourceLongitude=foobar sourceLatitude=foobar destinationLongitude=foobar destinationLatitude=foobar categoryDeviceType=foobar managerReceiptTime=foobar agentMacAddress=foobar$/m)
229
+ end
230
+
231
+ if ecs_select.active_mode != :disabled
232
+ let(:event_flat_hash) do
233
+ {
234
+ "[event][action]" => "floop", # deviceAction
235
+ "[network][protocol]" => "https", # applicationProtocol
236
+ "[cef][device_custom_ipv6_address_1][value]" => "4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9", # deviceCustomIPv6Address1
237
+ "[cef][device_custom_ipv6_address_1][label]" => "internal-interface", # deviceCustomIPv6Address1Label
238
+ "[observer][ip]" => "123.45.67.89", # deviceAddress
239
+ "[observer][hostname]" => "banana", # deviceHostName
240
+ "[user_agent][original]" => "'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='", # requestClientApplication
241
+ "[source][registered_domain]" => "monkey.see" # sourceDnsDomain
242
+ }
243
+ end
244
+
245
+ let(:event) do
246
+ event_flat_hash.each_with_object(LogStash::Event.new) do |(fr,v),memo|
247
+ memo.set(fr, v)
248
+ end
249
+ end
250
+
251
+ it 'encodes the ECS field names to their CEF name' do
252
+ codec.on_event{|data, newdata| results << newdata}
253
+ codec.fields = event_flat_hash.keys
254
+
255
+ codec.encode(event)
256
+
257
+ expect(results.first).to match(%r{^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=floop applicationProtocol=https deviceCustomIPv6Address1=4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9 deviceCustomIPv6Address1Label=internal-interface deviceAddress=123\.45\.67\.89 deviceHostName=banana requestClientApplication='Foo-Bar/2018\.1\.7; Email:user@example\.com; Guid:test\\=' sourceDnsDomain=monkey.see$}m)
258
+ end
259
+ end
260
+
261
+ context "with reverse_mapping set to true" do
262
+ subject(:codec) { LogStash::Codecs::CEF.new("reverse_mapping" => true) }
263
+
264
+ it "should encode the CEF field names to their short versions" do
265
+ codec.on_event{|data, newdata| results << newdata}
266
+ codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
267
+ event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
268
+ codec.encode(event)
269
+ expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=foobar app=foobar c6a1=foobar c6a1Label=foobar c6a2=foobar c6a2Label=foobar c6a3=foobar c6a3Label=foobar c6a4=foobar c6a4Label=foobar cat=foobar cfp1=foobar cfp1Label=foobar cfp2=foobar cfp2Label=foobar cfp3=foobar cfp3Label=foobar cfp4=foobar cfp4Label=foobar cn1=foobar cn1Label=foobar cn2=foobar cn2Label=foobar cn3=foobar cn3Label=foobar cnt=foobar cs1=foobar cs1Label=foobar cs2=foobar cs2Label=foobar cs3=foobar cs3Label=foobar cs4=foobar cs4Label=foobar cs5=foobar cs5Label=foobar cs6=foobar cs6Label=foobar dhost=foobar dmac=foobar dntdom=foobar dpid=foobar dpriv=foobar dproc=foobar dpt=foobar dst=foobar duid=foobar duser=foobar dvc=foobar dvchost=foobar dvcpid=foobar end=foobar fname=foobar fsize=foobar in=foobar msg=foobar out=foobar outcome=foobar proto=foobar request=foobar rt=foobar shost=foobar smac=foobar sntdom=foobar spid=foobar spriv=foobar sproc=foobar spt=foobar src=foobar start=foobar suid=foobar suser=foobar ahost=foobar art=foobar at=foobar aid=foobar agt=foobar av=foobar atz=foobar dtz=foobar slong=foobar slat=foobar dlong=foobar dlat=foobar catdt=foobar mrt=foobar amac=foobar$/m)
270
+ end
271
+
272
+ if ecs_select.active_mode != :disabled
273
+ let(:event_flat_hash) do
274
+ {
275
+ "[event][action]" => "floop", # act
276
+ "[network][protocol]" => "https", # app
277
+ "[cef][device_custom_ipv6_address_1][value]" => "4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9", # c6a1
278
+ "[cef][device_custom_ipv6_address_1][label]" => "internal-interface", # c6a1Label
279
+ "[observer][ip]" => "123.45.67.89", # dvc
280
+ "[observer][hostname]" => "banana", # dvchost
281
+ "[user_agent][original]" => "'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='",
282
+ "[source][registered_domain]" => "monkey.see" # sourceDnsDomain
283
+ }
284
+ end
285
+
286
+ let(:event) do
287
+ event_flat_hash.each_with_object(LogStash::Event.new) do |(fr,v),memo|
288
+ memo.set(fr, v)
289
+ end
290
+ end
291
+
292
+
293
+ it 'encodes the ECS field names to their CEF keys' do
294
+ codec.on_event{|data, newdata| results << newdata}
295
+ codec.fields = event_flat_hash.keys
296
+
297
+ codec.encode(event)
298
+
299
+ expect(results.first).to match(%r{^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=floop app=https c6a1=4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9 c6a1Label=internal-interface dvc=123\.45\.67\.89 dvchost=banana requestClientApplication='Foo-Bar/2018\.1\.7; Email:user@example\.com; Guid:test\\=' sourceDnsDomain=monkey.see$}m)
300
+ end
301
+ end
231
302
  end
232
303
  end
233
304
  end
@@ -305,11 +376,21 @@ describe LogStash::Codecs::CEF do
305
376
  end
306
377
  end
307
378
 
308
- context "#decode" do
309
- let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
310
-
379
+ module DecodeHelpers
311
380
  def validate(e)
312
381
  insist { e.is_a?(LogStash::Event) }
382
+ send("validate_ecs_#{ecs_compatibility}", e)
383
+ end
384
+
385
+ def validate_ecs_v1(e)
386
+ insist { e.get('[cef][version]') } == "0"
387
+ insist { e.get('[observer][version]') } == "1.0"
388
+ insist { e.get('[event][code]') } == "100"
389
+ insist { e.get('[cef][name]') } == "trojan successfully stopped"
390
+ insist { e.get('[event][severity]') } == "10"
391
+ end
392
+
393
+ def validate_ecs_disabled(e)
313
394
  insist { e.get('cefVersion') } == "0"
314
395
  insist { e.get('deviceVersion') } == "1.0"
315
396
  insist { e.get('deviceEventClassId') } == "100"
@@ -333,7 +414,11 @@ describe LogStash::Codecs::CEF do
333
414
  fail("Expected one event, got #{events.size} events: #{events.inspect}") unless events.size == 1
334
415
  event = events.first
335
416
 
336
- yield event if block_given?
417
+ if block_given?
418
+ aggregate_failures('decode one') do
419
+ yield event
420
+ end
421
+ end
337
422
 
338
423
  event
339
424
  end
@@ -359,369 +444,456 @@ describe LogStash::Codecs::CEF do
359
444
 
360
445
  events
361
446
  end
447
+ end
362
448
 
363
- context "with delimiter set" do
364
- # '\r\n' in single quotes to simulate the real input from a config
365
- # containing \r\n as 4-character sequence in the config:
366
- #
367
- # delimiter => "\r\n"
368
- #
369
- # Related: https://github.com/elastic/logstash/issues/1645
370
- subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
449
+ context "#decode", :ecs_compatibility_support do
450
+ ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
451
+ before(:each) do
452
+ allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
453
+ end
371
454
 
372
- it "should parse on the delimiter " do
373
- do_decode(subject,message) do |e|
374
- raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
375
- end
455
+ let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
376
456
 
377
- decode_one(subject, "\r\n") do |e|
378
- validate(e)
379
- insist { e.get("deviceVendor") } == "security"
380
- insist { e.get("deviceProduct") } == "threatmanager"
457
+ include DecodeHelpers
458
+
459
+ context "with delimiter set" do
460
+ # '\r\n' in single quotes to simulate the real input from a config
461
+ # containing \r\n as 4-character sequence in the config:
462
+ #
463
+ # delimiter => "\r\n"
464
+ #
465
+ # Related: https://github.com/elastic/logstash/issues/1645
466
+ subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
467
+
468
+ it "should parse on the delimiter " do
469
+ do_decode(subject,message) do |e|
470
+ raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
471
+ end
472
+
473
+ decode_one(subject, "\r\n") do |e|
474
+ validate(e)
475
+ insist { e.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "security"
476
+ insist { e.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "threatmanager"
477
+ end
381
478
  end
382
479
  end
383
- end
384
480
 
385
- context 'when a CEF header ends with a pair of properly-escaped backslashes' do
386
- let(:backslash) { '\\' }
387
- let(:pipe) { '|' }
388
- let(:message) { "CEF:0|security|threatmanager|1.0|100|double backslash" +
389
- backslash + backslash + # escaped backslash
390
- backslash + backslash + # escaped backslash
391
- "|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
481
+ context 'when a CEF header ends with a pair of properly-escaped backslashes' do
482
+ let(:backslash) { '\\' }
483
+ let(:pipe) { '|' }
484
+ let(:message) { "CEF:0|security|threatmanager|1.0|100|double backslash" +
485
+ backslash + backslash + # escaped backslash
486
+ backslash + backslash + # escaped backslash
487
+ "|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
392
488
 
393
- it 'should include the backslashes unescaped' do
394
- event = decode_one(subject, message)
489
+ it 'should include the backslashes unescaped' do
490
+ event = decode_one(subject, message)
395
491
 
396
- expect(event.get('name')).to eq('double backslash' + backslash + backslash )
397
- expect(event.get('severity')).to eq('10') # ensure we didn't consume the separator
492
+ expect(event.get(ecs_select[disabled:'name', v1:'[cef][name]'])).to eq('double backslash' + backslash + backslash )
493
+ expect(event.get(ecs_select[disabled:'severity',v1:'[event][severity]'])).to eq('10') # ensure we didn't consume the separator
494
+ end
398
495
  end
399
- end
400
496
 
401
- it "should parse the cef headers" do
402
- decode_one(subject, message) do |e|
403
- validate(e)
404
- insist { e.get("deviceVendor") } == "security"
405
- insist { e.get("deviceProduct") } == "threatmanager"
497
+ it "should parse the cef headers" do
498
+ decode_one(subject, message) do |e|
499
+ validate(e)
500
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == "security"
501
+ insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == "threatmanager"
502
+ end
406
503
  end
407
- end
408
504
 
409
- it "should parse the cef body" do
410
- decode_one(subject, message) do |e|
411
- insist { e.get("sourceAddress")} == "10.0.0.192"
412
- insist { e.get("destinationAddress") } == "12.121.122.82"
413
- insist { e.get("sourcePort") } == "1232"
505
+ it "should parse the cef body" do
506
+ decode_one(subject, message) do |e|
507
+ insist { e.get(ecs_select[disabled:"sourceAddress", v1:"[source][ip]"])} == "10.0.0.192"
508
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
509
+ insist { e.get(ecs_select[disabled:"sourcePort", v1:"[source][port]"]) } == "1232"
510
+ end
414
511
  end
415
- end
416
512
 
417
- let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
418
- it "should be OK with missing CEF headers (multiple pipes in sequence)" do
419
- decode_one(subject, missing_headers) do |e|
420
- validate(e)
421
- insist { e.get("deviceVendor") } == ""
422
- insist { e.get("deviceProduct") } == ""
513
+ let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
514
+ it "should be OK with missing CEF headers (multiple pipes in sequence)" do
515
+ decode_one(subject, missing_headers) do |e|
516
+ validate(e)
517
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == ""
518
+ insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == ""
519
+ end
423
520
  end
424
- end
425
521
 
426
- let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
427
- it "should strip leading whitespace from the message" do
428
- decode_one(subject, leading_whitespace) do |e|
429
- validate(e)
522
+ let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
523
+ it "should strip leading whitespace from the message" do
524
+ decode_one(subject, leading_whitespace) do |e|
525
+ validate(e)
526
+ end
430
527
  end
431
- end
432
528
 
433
- let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
434
- it "should be OK with escaped pipes in the message" do
435
- decode_one(subject, escaped_pipes) do |e|
436
- insist { e.get("moo") } == 'this\|has an escaped pipe'
529
+ let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
530
+ it "should be OK with escaped pipes in the message" do
531
+ decode_one(subject, escaped_pipes) do |e|
532
+ insist { e.get("moo") } == 'this\|has an escaped pipe'
533
+ end
437
534
  end
438
- end
439
535
 
440
- let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
441
- it "should be OK with not escaped pipes in the message" do
442
- decode_one(subject, pipes_in_message) do |e|
443
- insist { e.get("moo") } == 'this|has an pipe'
536
+ let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
537
+ it "should be OK with not escaped pipes in the message" do
538
+ decode_one(subject, pipes_in_message) do |e|
539
+ insist { e.get("moo") } == 'this|has an pipe'
540
+ end
444
541
  end
445
- end
446
542
 
447
- # while we may see these in practice, equals MUST be escaped in the extensions per the spec.
448
- let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
449
- it "should be OK with equal in the message" do
450
- decode_one(subject, equal_in_message) do |e|
451
- insist { e.get("moo") } == 'this =has = equals='
543
+ # while we may see these in practice, equals MUST be escaped in the extensions per the spec.
544
+ let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
545
+ it "should be OK with equal in the message" do
546
+ decode_one(subject, equal_in_message) do |e|
547
+ insist { e.get("moo") } == 'this =has = equals='
548
+ end
452
549
  end
453
- end
454
550
 
455
- let(:malformed_unescaped_equals_in_extension_value) { %q{CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar} }
456
- it 'should split correctly' do
457
- decode_one(subject, malformed_unescaped_equals_in_extension_value) do |event|
458
- expect(event.get('cefVersion')).to eq('0')
459
- expect(event.get('deviceVendor')).to eq('FooBar')
460
- expect(event.get('deviceProduct')).to eq('Web Gateway')
461
- expect(event.get('deviceVersion')).to eq('1.2.3.45.67')
462
- expect(event.get('deviceEventClassId')).to eq('200')
463
- expect(event.get('name')).to eq('Success')
464
- expect(event.get('severity')).to eq('2')
465
-
466
- # extension key/value pairs
467
- expect(event.get('deviceReceiptTime')).to eq('Sep 07 2018 14:50:39')
468
- expect(event.get('deviceEventCategory')).to eq('Access Log')
469
- expect(event.get('deviceVersion')).to eq('1.2.3.45.67')
470
- expect(event.get('destinationAddress')).to eq('1.1.1.1')
471
- expect(event.get('destinationHostName')).to eq('foo.example.com')
472
- expect(event.get('sourceUserName')).to eq('redacted')
473
- expect(event.get('sourceAddress')).to eq('2.2.2.2')
474
- expect(event.get('requestMethod')).to eq('POST')
475
- expect(event.get('requestUrl')).to eq(%q{'https://foo.example.com/bar/bingo/1'})
476
- # Although the value for `requestClientApplication` contains an illegal unquoted equals sign, the sequence
477
- # preceeding the unescaped-equals isn't shaped like a key, so we allow it to be a part of the value.
478
- expect(event.get('requestClientApplication')).to eq(%q{'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='})
479
- expect(event.get('deviceCustomString1Label')).to eq('Foo Bar')
480
- expect(event.get('deviceCustomString1')).to eq('')
551
+ context "zoneless deviceReceiptTime(rt) when deviceTimeZone(dtz) is provided" do
552
+ let(:cef_formatted_timestamp) { 'Jul 19 2017 10:50:21.127' }
553
+ let(:zone_name) { 'Europe/Moscow' }
554
+
555
+ let(:utc_timestamp) { Time.iso8601("2017-07-19T07:50:21.127Z") } # In summer of 2017, Europe/Moscow was UTC+03:00
556
+
557
+ let(:destination_time_zoned) { %Q{CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|Very-High| eventId=1 msg=Worm successfully stopped art=1500464384997 deviceSeverity=10 rt=#{cef_formatted_timestamp} src=10.0.0.1 sourceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 10.0.0.0-10.255.255.255 spt=1232 dst=2.1.2.2 destinationZoneURI=/All Zones/ArcSight System/Public Address Space Zones/RIPE NCC/2.0.0.0-2.255.255.255 (RIPE NCC) ahost=connector.rhel72 agt=192.168.231.129 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-51-8A-84 av=7.6.0.8009.0 atz=Europe/Lisbon at=syslog_file dvchost=client1 dtz=#{zone_name} _cefVer=0.1 aid=3UBajWl0BABCABBzZSlmUdw==} }
558
+
559
+ if ecs_select.active_mode == :disabled
560
+ it 'persists deviceReceiptTime and deviceTimeZone verbatim' do
561
+ decode_one(subject, destination_time_zoned) do |event|
562
+ expect(event.get('deviceReceiptTime')).to eq("Jul 19 2017 10:50:21.127")
563
+ expect(event.get('deviceTimeZone')).to eq('Europe/Moscow')
564
+ end
565
+ end
566
+ else
567
+ it 'sets the @timestamp using the value in `rt` combined with the offset provided by `dtz`' do
568
+ decode_one(subject, destination_time_zoned) do |event|
569
+ expected_time = LogStash::Timestamp.new(utc_timestamp)
570
+ expect(event.get('[@timestamp]').to_s).to eq(expected_time.to_s)
571
+ expect(event.get('[event][timezone]')).to eq(zone_name)
572
+ end
573
+ end
574
+ end
575
+ end
576
+
577
+ let(:malformed_unescaped_equals_in_extension_value) { %q{CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar} }
578
+ it 'should split correctly' do
579
+ decode_one(subject, malformed_unescaped_equals_in_extension_value) do |event|
580
+ expect(event.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"])).to eq('0')
581
+ expect(event.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"])).to eq('FooBar')
582
+ expect(event.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"])).to eq('Web Gateway')
583
+ expect(event.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"])).to eq('1.2.3.45.67')
584
+ expect(event.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"])).to eq('200')
585
+ expect(event.get(ecs_select[disabled:"name", v1:"[cef][name]"])).to eq('Success')
586
+ expect(event.get(ecs_select[disabled:"severity", v1:"[event][severity]"])).to eq('2')
587
+
588
+ # extension key/value pairs
589
+ if ecs_compatibility == :disabled
590
+ expect(event.get('deviceReceiptTime')).to eq('Sep 07 2018 14:50:39')
591
+ else
592
+ expected_time = LogStash::Timestamp.new(Time.parse('Sep 07 2018 14:50:39')).to_s
593
+ expect(event.get('[@timestamp]').to_s).to eq(expected_time)
594
+ end
595
+ expect(event.get(ecs_select[disabled:'deviceEventCategory', v1:'[cef][category]'])).to eq('Access Log')
596
+ expect(event.get(ecs_select[disabled:'deviceVersion', v1:'[observer][version]'])).to eq('1.2.3.45.67')
597
+ expect(event.get(ecs_select[disabled:'destinationAddress', v1:'[destination][ip]'])).to eq('1.1.1.1')
598
+ expect(event.get(ecs_select[disabled:'destinationHostName', v1:'[destination][domain]'])).to eq('foo.example.com')
599
+ expect(event.get(ecs_select[disabled:'sourceUserName', v1:'[source][user][name]'])).to eq('redacted')
600
+ expect(event.get(ecs_select[disabled:'sourceAddress', v1:'[source][ip]'])).to eq('2.2.2.2')
601
+ expect(event.get(ecs_select[disabled:'requestMethod', v1:'[http][request][method]'])).to eq('POST')
602
+ expect(event.get(ecs_select[disabled:'requestUrl', v1:'[url][original]'])).to eq(%q{'https://foo.example.com/bar/bingo/1'})
603
+ # Although the value for `requestClientApplication` contains an illegal unquoted equals sign, the sequence
604
+ # preceeding the unescaped-equals isn't shaped like a key, so we allow it to be a part of the value.
605
+ expect(event.get(ecs_select[disabled:'requestClientApplication',v1:'[user_agent][original]'])).to eq(%q{'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='})
606
+ expect(event.get(ecs_select[disabled:'deviceCustomString1Label',v1:'[cef][device_custom_string_1][label]'])).to eq('Foo Bar')
607
+ expect(event.get(ecs_select[disabled:'deviceCustomString1', v1:'[cef][device_custom_string_1][value]'])).to eq('')
608
+ end
481
609
  end
482
- end
483
610
 
484
- context('escaped-equals and unescaped-spaces in the extension values') do
485
- let(:query_string) { 'key1=value1&key2=value3 aa.bc&key3=value4'}
486
- let(:escaped_query_string) { query_string.gsub('=','\\=') }
487
- let(:cef_message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|go=start now query_string=#{escaped_query_string} final=done" }
611
+ context('escaped-equals and unescaped-spaces in the extension values') do
612
+ let(:query_string) { 'key1=value1&key2=value3 aa.bc&key3=value4'}
613
+ let(:escaped_query_string) { query_string.gsub('=','\\=') }
614
+ let(:cef_message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|go=start now query_string=#{escaped_query_string} final=done" }
488
615
 
489
- it 'captures the extension values correctly' do
490
- event = decode_one(subject, cef_message)
616
+ it 'captures the extension values correctly' do
617
+ event = decode_one(subject, cef_message)
491
618
 
492
- expect(event.get('go')).to eq('start now')
493
- expect(event.get('query_string')).to eq(query_string)
494
- expect(event.get('final')).to eq('done')
619
+ expect(event.get('go')).to eq('start now')
620
+ expect(event.get('query_string')).to eq(query_string)
621
+ expect(event.get('final')).to eq('done')
622
+ end
495
623
  end
496
- end
497
624
 
498
- let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
499
- it "should be OK with escaped backslash in the headers" do
500
- decode_one(subject, escaped_backslash_in_header) do |e|
501
- insist { e.get("cefVersion") } == '0'
502
- insist { e.get("deviceVendor") } == 'secu\\rity'
503
- insist { e.get("deviceProduct") } == 'threat\\manager'
504
- insist { e.get("deviceVersion") } == '1.\\0'
505
- insist { e.get("deviceEventClassId") } == '10\\0'
506
- insist { e.get("name") } == 'tro\\jan successfully stopped'
507
- insist { e.get("severity") } == '\\10'
625
+ let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
626
+ it "should be OK with escaped backslash in the headers" do
627
+ decode_one(subject, escaped_backslash_in_header) do |e|
628
+ insist { e.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"]) } == '0'
629
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'secu\\rity'
630
+ insist { e.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"]) } == 'threat\\manager'
631
+ insist { e.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"]) } == '1.\\0'
632
+ insist { e.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"]) } == '10\\0'
633
+ insist { e.get(ecs_select[disabled:"name", v1:"[cef][name]"]) } == 'tro\\jan successfully stopped'
634
+ insist { e.get(ecs_select[disabled:"severity", v1:"[event][severity]"]) } == '\\10'
635
+ end
508
636
  end
509
- end
510
637
 
511
- let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
512
- it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
513
- decode_one(subject, escaped_backslash_in_header_edge_case) do |e|
514
- validate(e)
515
- insist { e.get("deviceVendor") } == 'security\\|'
516
- insist { e.get("deviceProduct") } == 'threatmanager\\'
638
+ let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
639
+ it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
640
+ decode_one(subject, escaped_backslash_in_header_edge_case) do |e|
641
+ validate(e)
642
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'security\\|'
643
+ insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == 'threatmanager\\'
644
+ end
517
645
  end
518
- end
519
646
 
520
- let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
521
- it "should be OK with escaped pipes in the headers" do
522
- decode_one(subject, escaped_pipes_in_header) do |e|
523
- insist { e.get("cefVersion") } == '0'
524
- insist { e.get("deviceVendor") } == 'secu|rity'
525
- insist { e.get("deviceProduct") } == 'threatmanager|'
526
- insist { e.get("deviceVersion") } == '1.|0'
527
- insist { e.get("deviceEventClassId") } == '10|0'
528
- insist { e.get("name") } == 'tro|jan successfully stopped'
529
- insist { e.get("severity") } == '|10'
647
+ let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
648
+ it "should be OK with escaped pipes in the headers" do
649
+ decode_one(subject, escaped_pipes_in_header) do |e|
650
+ insist { e.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"]) } == '0'
651
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'secu|rity'
652
+ insist { e.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"]) } == 'threatmanager|'
653
+ insist { e.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"]) } == '1.|0'
654
+ insist { e.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"]) } == '10|0'
655
+ insist { e.get(ecs_select[disabled:"name", v1:"[cef][name]"]) } == 'tro|jan successfully stopped'
656
+ insist { e.get(ecs_select[disabled:"severity", v1:"[event][severity]"]) } == '|10'
657
+ end
530
658
  end
531
- end
532
659
 
533
- let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
534
- it "should be OK with backslashs in the message" do
535
- decode_one(subject, backslash_in_message) do |e|
536
- insist { e.get("moo") } == 'this \\has \\ backslashs\\'
660
+ let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
661
+ it "should be OK with backslashs in the message" do
662
+ decode_one(subject, backslash_in_message) do |e|
663
+ insist { e.get("moo") } == 'this \\has \\ backslashs\\'
664
+ end
537
665
  end
538
- end
539
666
 
540
- let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
541
- it "should be OK with equal in the headers" do
542
- decode_one(subject, equal_in_header) do |e|
543
- validate(e)
544
- insist { e.get("deviceProduct") } == "threatmanager=equal"
667
+ let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
668
+ it "should be OK with equal in the headers" do
669
+ decode_one(subject, equal_in_header) do |e|
670
+ validate(e)
671
+ insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == "threatmanager=equal"
672
+ end
545
673
  end
546
- end
547
674
 
548
- let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232'}
549
- it "should be OK to have one or more spaces between keys" do
550
- decode_one(subject, spaces_in_between_keys) do |e|
551
- validate(e)
552
- insist { e.get("sourceAddress") } == "10.0.0.192"
553
- insist { e.get("destinationAddress") } == "12.121.122.82"
554
- insist { e.get("sourcePort") } == "1232"
675
+ let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232'}
676
+ it "should be OK to have one or more spaces between keys" do
677
+ decode_one(subject, spaces_in_between_keys) do |e|
678
+ validate(e)
679
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
680
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
681
+ insist { e.get(ecs_select[disabled:"sourcePort",v1:"[source][port]"]) } == "1232"
682
+ end
555
683
  end
556
- end
557
684
 
558
- let (:dots_in_keys) {'CEF:0|Vendor|Device|Version|13|my message|5|dvchost=loghost cat=traffic deviceSeverity=notice ad.nn=TEST src=192.168.0.1 destinationPort=53'}
559
- it "should be OK with dots in keys" do
560
- decode_one(subject, dots_in_keys) do |e|
561
- insist { e.get("deviceHostName") } == "loghost"
562
- insist { e.get("ad.nn") } == 'TEST'
563
- insist { e.get("sourceAddress") } == '192.168.0.1'
564
- insist { e.get("destinationPort") } == '53'
685
+ let (:dots_in_keys) {'CEF:0|Vendor|Device|Version|13|my message|5|dvchost=loghost cat=traffic deviceSeverity=notice ad.nn=TEST src=192.168.0.1 destinationPort=53'}
686
+ it "should be OK with dots in keys" do
687
+ decode_one(subject, dots_in_keys) do |e|
688
+ insist { e.get(ecs_select[disabled:"deviceHostName",v1:"[observer][hostname]"]) } == "loghost"
689
+ insist { e.get("ad.nn") } == 'TEST'
690
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == '192.168.0.1'
691
+ insist { e.get(ecs_select[disabled:"destinationPort",v1:"[destination][port]"]) } == '53'
692
+ end
565
693
  end
566
- end
567
694
 
568
- let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 dproc=InternetExplorer x.x.x.x'}
569
- it "should be OK to have one or more spaces in values" do
570
- decode_one(subject, allow_spaces_in_values) do |e|
571
- validate(e)
572
- insist { e.get("sourceAddress") } == "10.0.0.192"
573
- insist { e.get("destinationAddress") } == "12.121.122.82"
574
- insist { e.get("sourcePort") } == "1232"
575
- insist { e.get("destinationProcessName") } == "InternetExplorer x.x.x.x"
695
+ let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 dproc=InternetExplorer x.x.x.x'}
696
+ it "should be OK to have one or more spaces in values" do
697
+ decode_one(subject, allow_spaces_in_values) do |e|
698
+ validate(e)
699
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
700
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
701
+ insist { e.get(ecs_select[disabled:"sourcePort",v1:"[source][port]"]) } == "1232"
702
+ insist { e.get(ecs_select[disabled:"destinationProcessName",v1:"[destination][process][name]"]) } == "InternetExplorer x.x.x.x"
703
+ end
576
704
  end
577
- end
578
705
 
579
- let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
580
- it "should keep ad.fields" do
581
- decode_one(subject, preserve_additional_fields_with_dot_notations) do |e|
582
- validate(e)
583
- insist { e.get("sourceAddress") } == "10.0.0.192"
584
- insist { e.get("destinationAddress") } == "12.121.122.82"
585
- insist { e.get("[ad.field][0]") } == "field0"
586
- insist { e.get("[ad.name][1]") } == "new_name"
587
- insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
588
- insist { e.get('ad.Error_,Code') } == "3221225578"
589
- insist { e.get("additional.dotfieldName") } == "new_value"
706
+ let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
707
+ it "should keep ad.fields" do
708
+ decode_one(subject, preserve_additional_fields_with_dot_notations) do |e|
709
+ validate(e)
710
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
711
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
712
+ insist { e.get("[ad.field][0]") } == "field0"
713
+ insist { e.get("[ad.name][1]") } == "new_name"
714
+ insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
715
+ insist { e.get('ad.Error_,Code') } == "3221225578"
716
+ insist { e.get("additional.dotfieldName") } == "new_value"
717
+ end
590
718
  end
591
- end
592
719
 
593
- let(:preserve_complex_multiple_dot_notation_in_extension_fields) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name' }
594
- it "should keep ad.fields" do
595
- decode_one(subject, preserve_complex_multiple_dot_notation_in_extension_fields) do |e|
596
- validate(e)
597
- insist { e.get("sourceAddress") } == "10.0.0.192"
598
- insist { e.get("destinationAddress") } == "12.121.122.82"
599
- insist { e.get("[ad.field][0]") } == "field0"
600
- insist { e.get("[ad.foo.name][1]") } == "new_name"
601
- insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
602
- insist { e.get('ad.Error_,Code') } == "3221225578"
603
- insist { e.get("additional.dotfieldName") } == "new_value"
720
+ let(:preserve_complex_multiple_dot_notation_in_extension_fields) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name' }
721
+ it "should keep ad.fields" do
722
+ decode_one(subject, preserve_complex_multiple_dot_notation_in_extension_fields) do |e|
723
+ validate(e)
724
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
725
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
726
+ insist { e.get("[ad.field][0]") } == "field0"
727
+ insist { e.get("[ad.foo.name][1]") } == "new_name"
728
+ insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
729
+ insist { e.get('ad.Error_,Code') } == "3221225578"
730
+ insist { e.get("additional.dotfieldName") } == "new_value"
731
+ end
604
732
  end
605
- end
606
733
 
607
- let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
608
- it "should correctly parse random values even with additional fields in message" do
609
- decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
610
- validate(e)
611
- insist { e.get("sourceAddress") } == "10.0.0.192"
612
- insist { e.get("destinationAddress") } == "12.121.122.82"
613
- insist { e.get("[ad.field][0]") } == "field0"
614
- insist { e.get("[ad.name][1]") } == "new_name"
615
- insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
616
- insist { e.get("ad.Error_,Code") } == "3221225578"
617
- insist { e.get("additional.dotfieldName") } == "new_value"
618
- insist { e.get("deviceCustomString4") } == "401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34"
734
+ let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
735
+ it "should correctly parse random values even with additional fields in message" do
736
+ decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
737
+ validate(e)
738
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
739
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
740
+ insist { e.get("[ad.field][0]") } == "field0"
741
+ insist { e.get("[ad.name][1]") } == "new_name"
742
+ insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
743
+ insist { e.get("ad.Error_,Code") } == "3221225578"
744
+ insist { e.get("additional.dotfieldName") } == "new_value"
745
+ insist { e.get(ecs_select[disabled:"deviceCustomString4",v1:"[cef][device_custom_string_4][value]"]) } == "401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34"
746
+ end
619
747
  end
620
- end
621
748
 
622
- let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
623
- it "should preserve unmatched key mappings" do
624
- decode_one(subject, preserve_unmatched_key_mappings) do |e|
625
- validate(e)
626
- insist { e.get("sourceAddress") } == "10.0.0.192"
627
- insist { e.get("destinationAddress") } == "12.121.122.82"
628
- insist { e.get("new_key_by_device") } == "new_values here"
749
+ let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
750
+ it "should preserve unmatched key mappings" do
751
+ decode_one(subject, preserve_unmatched_key_mappings) do |e|
752
+ validate(e)
753
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
754
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
755
+ insist { e.get("new_key_by_device") } == "new_values here"
756
+ end
629
757
  end
630
- end
631
758
 
632
- let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
633
- it "should translate most known abbreviated CEF field names" do
634
- decode_one(subject, translate_abbreviated_cef_fields) do |e|
635
- validate(e)
636
- insist { e.get("sourceAddress") } == "10.0.0.192"
637
- insist { e.get("destinationAddress") } == "12.121.122.82"
638
- insist { e.get("transportProtocol") } == "TCP"
639
- insist { e.get("sourceHostName") } == "source.host.name"
640
- insist { e.get("destinationHostName") } == "destination.host.name"
641
- insist { e.get("sourcePort") } == "11024"
642
- insist { e.get("destinationPort") } == "9200"
643
- insist { e.get("eventOutcome") } == "Success"
644
- insist { e.get("agentMacAddress")} == "00:80:48:1c:24:91"
759
+ let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
760
+ it "should translate most known abbreviated CEF field names" do
761
+ decode_one(subject, translate_abbreviated_cef_fields) do |e|
762
+ validate(e)
763
+ insist { e.get(ecs_select[disabled:"sourceAddress", v1:"[source][ip]"]) } == "10.0.0.192"
764
+ insist { e.get(ecs_select[disabled:"destinationAddress", v1:"[destination][ip]"]) } == "12.121.122.82"
765
+ insist { e.get(ecs_select[disabled:"transportProtocol", v1:"[network][transport]"]) } == "TCP"
766
+ insist { e.get(ecs_select[disabled:"sourceHostName", v1:"[source][domain]"]) } == "source.host.name"
767
+ insist { e.get(ecs_select[disabled:"destinationHostName",v1:"[destination][domain]"]) } == "destination.host.name"
768
+ insist { e.get(ecs_select[disabled:"sourcePort", v1:"[source][port]"]) } == "11024"
769
+ insist { e.get(ecs_select[disabled:"destinationPort", v1:"[destination][port]"]) } == "9200"
770
+ insist { e.get(ecs_select[disabled:"eventOutcome", v1:"[event][outcome]"]) } == "Success"
771
+ insist { e.get(ecs_select[disabled:"agentMacAddress", v1:"[agent][mac]"])} == "00:80:48:1c:24:91"
772
+ end
645
773
  end
646
- end
647
774
 
648
- let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
649
- it "Should detect headers before CEF starts" do
650
- decode_one(subject, syslog) do |e|
651
- validate(e)
652
- insist { e.get('syslog') } == 'Syslogdate Sysloghost'
775
+ let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
776
+ it "Should detect headers before CEF starts" do
777
+ decode_one(subject, syslog) do |e|
778
+ validate(e)
779
+ insist { e.get(ecs_select[disabled:'syslog',v1:'[log][syslog][header]']) } == 'Syslogdate Sysloghost'
780
+ end
653
781
  end
654
- end
655
782
 
656
- context 'with UTF-8 message' do
657
- let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted' }
658
-
659
- # since this spec is encoded UTF-8, the literal strings it contains are encoded with UTF-8,
660
- # but codecs in Logstash tend to receive their input as BINARY (or: ASCII-8BIT); ensure that
661
- # we can handle either without losing the UTF-8 characters from the higher planes.
662
- %w(
663
- BINARY
664
- UTF-8
665
- ).each do |external_encoding|
666
- context "externally encoded as #{external_encoding}" do
667
- let(:message) { super().force_encoding(external_encoding) }
668
- it 'should keep the higher-plane characters' do
669
- decode_one(subject, message.dup) do |event|
670
- validate(event)
671
- insist { event.get("target") } == "aaaaaああああaaaa"
672
- insist { event.get("target").encoding } == Encoding::UTF_8
783
+ context 'with UTF-8 message' do
784
+ let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted' }
785
+
786
+ # since this spec is encoded UTF-8, the literal strings it contains are encoded with UTF-8,
787
+ # but codecs in Logstash tend to receive their input as BINARY (or: ASCII-8BIT); ensure that
788
+ # we can handle either without losing the UTF-8 characters from the higher planes.
789
+ %w(
790
+ BINARY
791
+ UTF-8
792
+ ).each do |external_encoding|
793
+ context "externally encoded as #{external_encoding}" do
794
+ let(:message) { super().force_encoding(external_encoding) }
795
+ it 'should keep the higher-plane characters' do
796
+ decode_one(subject, message.dup) do |event|
797
+ validate(event)
798
+ insist { event.get("target") } == "aaaaaああああaaaa"
799
+ insist { event.get("target").encoding } == Encoding::UTF_8
800
+ end
673
801
  end
674
802
  end
675
803
  end
676
804
  end
677
- end
678
805
 
679
- context 'non-UTF-8 message' do
680
- let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
681
- it 'should emit message unparsed with _cefparsefailure tag' do
682
- decode_one(subject, message.dup) do |event|
683
- insist { event.get("message").bytes.to_a } == message.bytes.to_a
684
- insist { event.get("tags") } == ['_cefparsefailure']
806
+ context 'non-UTF-8 message' do
807
+ let(:logger_stub) { double('Logger').as_null_object }
808
+ before(:each) do
809
+ allow_any_instance_of(described_class).to receive(:logger).and_return(logger_stub)
810
+ end
811
+ let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
812
+ it 'should emit message unparsed with _cefparsefailure tag' do
813
+ decode_one(subject, message.dup) do |event|
814
+ insist { event.get("message").bytes.to_a } == message.bytes.to_a
815
+ insist { event.get("tags") } == ['_cefparsefailure']
816
+ end
817
+ expect(logger_stub).to have_received(:error).with(/Failed to decode CEF payload/, any_args)
685
818
  end
686
819
  end
687
- end
688
820
 
689
- context "with raw_data_field set" do
690
- subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
821
+ context "with raw_data_field set" do
822
+ subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
691
823
 
692
- it "should return the raw message in field message_raw" do
693
- decode_one(subject, message.dup) do |e|
694
- validate(e)
695
- insist { e.get("message_raw") } == message
824
+ it "should return the raw message in field message_raw" do
825
+ decode_one(subject, message.dup) do |e|
826
+ validate(e)
827
+ insist { e.get("message_raw") } == message
828
+ end
829
+ end
830
+ end
831
+
832
+ context "legacy aliases" do
833
+ let(:cef_line) { "CEF:0|security|threatmanager|1.0|100|target acquired|10|destinationLongitude=-73.614830 destinationLatitude=45.505918 sourceLongitude=45.4628328 sourceLatitude=9.1076927" }
834
+
835
+ it ecs_select[disabled:"creates the fields as provided",v1:"maps to ECS fields"] do
836
+ decode_one(codec, cef_line.dup) do |event|
837
+ # |---- LEGACY: AS-PROVIDED ----| |--------- ECS: MAP TO FIELD ----------|
838
+ expect(event.get(ecs_select[disabled:'destinationLongitude',v1:'[destination][geo][location][lon]'])).to eq('-73.614830')
839
+ expect(event.get(ecs_select[disabled:'destinationLatitude', v1:'[destination][geo][location][lat]'])).to eq('45.505918')
840
+ expect(event.get(ecs_select[disabled:'sourceLongitude', v1:'[source][geo][location][lon]' ])).to eq('45.4628328')
841
+ expect(event.get(ecs_select[disabled:'sourceLatitude', v1:'[source][geo][location][lat]' ])).to eq('9.1076927')
842
+ end
696
843
  end
697
844
  end
698
845
  end
699
846
  end
700
847
 
701
- context "encode and decode" do
848
+ context "encode and decode", :ecs_compatibility_support do
702
849
  subject(:codec) { LogStash::Codecs::CEF.new }
703
850
 
704
851
  let(:results) { [] }
705
852
 
706
- it "should return an equal event if encoded and decoded again" do
707
- codec.on_event{|data, newdata| results << newdata}
708
- codec.vendor = "%{deviceVendor}"
709
- codec.product = "%{deviceProduct}"
710
- codec.version = "%{deviceVersion}"
711
- codec.signature = "%{deviceEventClassId}"
712
- codec.name = "%{name}"
713
- codec.severity = "%{severity}"
714
- codec.fields = [ "foo" ]
715
- event = LogStash::Event.new("deviceVendor" => "vendor", "deviceProduct" => "product", "deviceVersion" => "2.0", "deviceEventClassId" => "signature", "name" => "name", "severity" => "1", "foo" => "bar")
716
- codec.encode(event)
717
- codec.decode(results.first) do |e|
718
- expect(e.get('deviceVendor')).to be == event.get('deviceVendor')
719
- expect(e.get('deviceProduct')).to be == event.get('deviceProduct')
720
- expect(e.get('deviceVersion')).to be == event.get('deviceVersion')
721
- expect(e.get('deviceEventClassId')).to be == event.get('deviceEventClassId')
722
- expect(e.get('name')).to be == event.get('name')
723
- expect(e.get('severity')).to be == event.get('severity')
724
- expect(e.get('foo')).to be == event.get('foo')
853
+ ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
854
+ before(:each) do
855
+ allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
856
+ end
857
+
858
+ let(:vendor_field) { ecs_select[disabled:'deviceVendor', v1:'[observer][vendor]'] }
859
+ let(:product_field) { ecs_select[disabled:'deviceProduct', v1:'[observer][product]']}
860
+ let(:version_field) { ecs_select[disabled:'deviceVersion', v1:'[observer][version]']}
861
+ let(:signature_field) { ecs_select[disabled:'deviceEventClassId', v1:'[event][code]']}
862
+ let(:name_field) { ecs_select[disabled:'name', v1:'[cef][name]']}
863
+ let(:severity_field) { ecs_select[disabled:'severity', v1:'[event][severity]']}
864
+
865
+ let(:source_dns_domain_field) { ecs_select[disabled:'sourceDnsDomain',v1:'[source][registered_domain]'] }
866
+
867
+ it "should return an equal event if encoded and decoded again" do
868
+ codec.on_event{|data, newdata| results << newdata}
869
+ codec.vendor = "%{" + vendor_field + "}"
870
+ codec.product = "%{" + product_field + "}"
871
+ codec.version = "%{" + version_field + "}"
872
+ codec.signature = "%{" + signature_field + "}"
873
+ codec.name = "%{" + name_field + "}"
874
+ codec.severity = "%{" + severity_field + "}"
875
+ codec.fields = [ "foo", source_dns_domain_field ]
876
+ event = LogStash::Event.new.tap do |e|
877
+ e.set(vendor_field, "vendor")
878
+ e.set(product_field, "product")
879
+ e.set(version_field, "2.0")
880
+ e.set(signature_field, "signature")
881
+ e.set(name_field, "name")
882
+ e.set(severity_field, "1")
883
+ e.set("foo", "bar")
884
+ e.set(source_dns_domain_field, "apple")
885
+ end
886
+ codec.encode(event)
887
+ codec.decode(results.first) do |e|
888
+ expect(e.get(vendor_field)).to be == event.get(vendor_field)
889
+ expect(e.get(product_field)).to be == event.get(product_field)
890
+ expect(e.get(version_field)).to be == event.get(version_field)
891
+ expect(e.get(signature_field)).to be == event.get(signature_field)
892
+ expect(e.get(name_field)).to be == event.get(name_field)
893
+ expect(e.get(severity_field)).to be == event.get(severity_field)
894
+ expect(e.get('foo')).to be == event.get('foo')
895
+ expect(e.get(source_dns_domain_field)).to be == event.get(source_dns_domain_field)
896
+ end
725
897
  end
726
898
  end
727
899
  end