logstash-codec-cef 6.1.2-java → 6.2.3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +15 -1
  3. data/docs/index.asciidoc +452 -9
  4. data/lib/logstash/codecs/cef.rb +272 -111
  5. data/lib/logstash/codecs/cef/timestamp_normalizer.rb +115 -0
  6. data/logstash-codec-cef.gemspec +3 -1
  7. data/spec/codecs/cef/timestamp_normalizer_spec.rb +274 -0
  8. data/spec/codecs/cef_spec.rb +500 -306
  9. metadata +37 -7
data/lib/logstash/codecs/cef/timestamp_normalizer.rb ADDED
@@ -0,0 +1,115 @@
1
+ # encoding: utf-8
2
+
3
+ require 'java'
4
+
5
+ # The CEF specification allows a variety of timestamp formats, some of which
6
+ # cannot be unambiguously parsed to a specific points in time, and may require
7
+ # additional side-channel information to do so, namely:
8
+ # - the time zone or UTC offset (which MAY be included in a separate field)
9
+ # - the locale (for parsing abbreviated month names)
10
+ # - the year (assume "recent")
11
+ #
12
+ # This normalizer attempts to use the provided context and make reasonable
13
+ # assumptions when parsing ambiguous dates.
14
+ class LogStash::Codecs::CEF::TimestampNormalizer
15
+
16
+ java_import java.time.Clock
17
+ java_import java.time.LocalDate
18
+ java_import java.time.LocalTime
19
+ java_import java.time.MonthDay
20
+ java_import java.time.OffsetDateTime
21
+ java_import java.time.ZoneId
22
+ java_import java.time.ZonedDateTime
23
+ java_import java.time.format.DateTimeFormatter
24
+ java_import java.util.Locale
25
+
26
+ def initialize(locale:nil, timezone:nil, clock: Clock.systemUTC)
27
+ @clock = clock
28
+
29
+ java_locale = locale ? get_locale(locale) : Locale.get_default
30
+ java_timezone = timezone ? ZoneId.of(timezone) : ZoneId.system_default
31
+
32
+ @cef_timestamp_format_parser = DateTimeFormatter
33
+ .ofPattern("MMM dd[ yyyy] HH:mm:ss[.SSSSSSSSS][.SSSSSS][.SSS][ zzz]")
34
+ .withZone(java_timezone)
35
+ .withLocale(java_locale)
36
+ end
37
+
38
+ INTEGER_OR_DECIMAL_PATTERN = /\A[1-9][0-9]*(?:\.[0-9]+)?\z/
39
+ private_constant :INTEGER_OR_DECIMAL_PATTERN
40
+
41
+ # @param value [String,Time,Numeric]
42
+ # The value to parse. `Time`s are returned without modification, and `Numeric` values
43
+ # are treated as millis-since-epoch (as are fully-numeric strings).
44
+ # Strings are parsed unsing any of the supported CEF formats, and when the timestamp
45
+ # does not encode a year, we assume the year from contextual information like the
46
+ # current time.
47
+ # @param device_timezone_name [String,nil] (optional):
48
+ # If known, the time-zone or UTC offset of the device that encoded the timestamp.
49
+ # This value is used to determine the offset when no offset is encoded in the timestamp.
50
+ # If not provided, the system default time zone is used instead.
51
+ # @return [Time]
52
+ def normalize(value, device_timezone_name=nil)
53
+ return value if value.kind_of?(Time)
54
+
55
+ case value
56
+ when Numeric then Time.at(Rational(value, 1000))
57
+ when INTEGER_OR_DECIMAL_PATTERN then Time.at(Rational(value, 1000))
58
+ else
59
+ parse_cef_format_string(value.to_s, device_timezone_name)
60
+ end
61
+ end
62
+
63
+ private
64
+
65
+ def get_locale(spec)
66
+ if spec.nil?
67
+ Locale.get_default
68
+ elsif spec =~ /\A([a-z]{2})_([A-Z]{2})\z/
69
+ lang, country = Regexp.last_match(1), Regexp.last_match(2)
70
+ Locale.new(lang, country)
71
+ else
72
+ Locale.for_language_tag(spec)
73
+ end
74
+ end
75
+
76
+ def parse_cef_format_string(value, context_timezone=nil)
77
+ cef_timestamp_format_parser = @cef_timestamp_format_parser
78
+ cef_timestamp_format_parser = cef_timestamp_format_parser.with_zone(java.time.ZoneId.of(context_timezone)) unless context_timezone.nil?
79
+
80
+ parsed_time = cef_timestamp_format_parser.parse_best(value,
81
+ ->(v){ ZonedDateTime.from(v) },
82
+ ->(v){ OffsetDateTime.from(v) },
83
+ ->(v){ resolve_assuming_year(v) }).to_instant
84
+
85
+ # Ruby's `Time::at(sec, microseconds_with_frac)`
86
+ Time.at(parsed_time.get_epoch_second, Rational(parsed_time.get_nano, 1000))
87
+ rescue => e
88
+ $stderr.puts "parse_cef_format_sgring(#{value.inspect}, #{context_timezone.inspect}) #!=> #{e.message}"
89
+ raise
90
+ end
91
+
92
+ def resolve_assuming_year(parsed_temporal_accessor)
93
+ parsed_monthday = MonthDay.from(parsed_temporal_accessor)
94
+ parsed_time = LocalTime.from(parsed_temporal_accessor)
95
+ parsed_zone = ZoneId.from(parsed_temporal_accessor)
96
+
97
+ now = ZonedDateTime.now(@clock.with_zone(parsed_zone))
98
+
99
+ parsed_timestamp_with_current_year = ZonedDateTime.of(parsed_monthday.at_year(now.get_year), parsed_time, parsed_zone)
100
+
101
+ if (parsed_timestamp_with_current_year > now.plus_days(2))
102
+ # e.g., on May 12, parsing a date from May 15 or later is plausibly from
103
+ # the prior calendar year and not actually from the future
104
+ return ZonedDateTime.of(parsed_monthday.at_year(now.get_year - 1), parsed_time, parsed_zone)
105
+ elsif now.get_month_value == 12 && (parsed_timestamp_with_current_year.plus_years(1) <= now.plus_days(2))
106
+ # e.g., on December 31, parsing a date from January 1 could plausibly be
107
+ # from the very-near future but next calendar year due to out-of-sync
108
+ # clocks, mismatched timezones, etc.
109
+ return ZonedDateTime.of(parsed_monthday.at_year(now.get_year + 1), parsed_time, parsed_zone)
110
+ else
111
+ # otherwise, assume current calendar year
112
+ return parsed_timestamp_with_current_year
113
+ end
114
+ end
115
+ end
data/logstash-codec-cef.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-cef'
4
- s.version = '6.1.2'
4
+ s.version = '6.2.3'
5
5
  s.platform = 'java'
6
6
  s.licenses = ['Apache License (2.0)']
7
7
  s.summary = "Reads the ArcSight Common Event Format (CEF)."
@@ -22,6 +22,8 @@ Gem::Specification.new do |s|
22
22
 
23
23
  # Gem dependencies
24
24
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
25
+ s.add_runtime_dependency "logstash-mixin-ecs_compatibility_support", '~> 1.3'
26
+ s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
25
27
 
26
28
  s.add_development_dependency 'logstash-devutils'
27
29
  s.add_development_dependency 'insist'
data/spec/codecs/cef/timestamp_normalizer_spec.rb ADDED
@@ -0,0 +1,274 @@
1
+ # encoding: utf-8
2
+
3
+ require 'logstash/util'
4
+ require "logstash/devutils/rspec/spec_helper"
5
+ require "insist"
6
+ require "logstash/codecs/cef"
7
+ require 'logstash/codecs/cef/timestamp_normalizer'
8
+
9
+ describe LogStash::Codecs::CEF::TimestampNormalizer do
10
+
11
+ subject(:timestamp_normalizer) { described_class.new }
12
+ let(:parsed_result) { timestamp_normalizer.normalize(parsable_string) }
13
+
14
+ context "parsing dates with a year specified" do
15
+ let(:parsable_string) { "Jun 17 2027 17:57:06.456" }
16
+ it 'parses the year correctly' do
17
+ expect(parsed_result.year).to eq(2027)
18
+ end
19
+ end
20
+
21
+ context "unparsable inputs" do
22
+ let(:parsable_string) { "Last Thursday" }
23
+ it "raises a StandardError exception that can be caught upstream" do
24
+ expect { parsed_result }.to raise_error(StandardError, /#{Regexp::escape parsable_string}/)
25
+ end
26
+ end
27
+
28
+ context "side-channel time zone indicators" do
29
+ let(:context_timezone) { 'America/New_York' }
30
+ let(:parsed_result) { timestamp_normalizer.normalize(parsable_string, context_timezone) }
31
+
32
+ context "when parsed input does not include offset information" do
33
+ let(:parsable_string) { "Jun 17 2027 17:57:06.456" }
34
+
35
+ it 'offsets to the context timezone time' do
36
+ expect(parsed_result).to eq(Time.parse("2027-06-17T21:57:06.456Z"))
37
+ end
38
+ end
39
+ context "when parsed input includes offset information" do
40
+ let(:parsable_string) { "Jun 17 2027 17:57:06.456 -07:00" }
41
+
42
+ it 'uses the parsed offset' do
43
+ expect(parsed_result).to eq(Time.parse("2027-06-18T00:57:06.456Z"))
44
+ end
45
+ end
46
+ context "when parsed input is a millis-since-epoch timestamp" do
47
+ let(:parsable_string) { "1616623591694" }
48
+
49
+ it "does not offset the time" do
50
+ expect(parsed_result).to eq(Time.at(Rational(1616623591694,1_000)))
51
+ expect(parsed_result.nsec).to eq(694_000_000)
52
+ end
53
+ end
54
+ context "when parsed input is a millis-since-epoch timestamp with decimal part and microsecond precision" do
55
+ let(:parsable_string) { "1616623591694.176" }
56
+
57
+ it "does not offset the time" do
58
+ expect(parsed_result).to eq(Time.at(Rational(1616623591694176,1_000_000)))
59
+ expect(parsed_result.nsec).to eq(694_176_000)
60
+ end
61
+ end
62
+ context "when parsed input is a millis-since-epoch timestamp with decimal part and nanosecond precision" do
63
+ let(:parsable_string) { "1616623591694.176789" }
64
+
65
+ it "does not offset the time" do
66
+ expect(parsed_result).to eq(Time.at(Rational(1616623591694176789,1_000_000_000)))
67
+ expect(parsed_result.nsec).to eq(694_176_789)
68
+ end
69
+ end
70
+ end
71
+
72
+ context "when locale is specified" do
73
+ let(:locale_language) { 'de' }
74
+ let(:locale_country) { 'DE' }
75
+ let(:locale_spec) { "#{locale_language}_#{locale_country}" }
76
+
77
+ # Due to locale-provider loading changes in JDK 9, abbreviations for months
78
+ # depend on a combination of the JDK version and the `java.locale.providers`
79
+ # system property.
80
+ # Instead of hard-coding a localized month name, use this process's locales
81
+ # to generate one.
82
+ let(:java_locale) { java.util.Locale.new(locale_language, locale_country) }
83
+ let(:localized_march_abbreviation) do
84
+ months = java.text.DateFormatSymbols.new(java_locale).get_short_months
85
+ months[2] # march
86
+ end
87
+
88
+ subject(:timestamp_normalizer) { described_class.new(locale: locale_spec) }
89
+
90
+ let(:parsable_string) { "#{localized_march_abbreviation} 17 2019 17:57:06.456 +01:00" }
91
+
92
+ it 'uses the locale to parse the date' do
93
+ expect(parsed_result).to eq(Time.parse("2019-03-17T17:57:06.456+01:00"))
94
+ end
95
+ end
96
+
97
+ context "parsing dates with sub-second precision" do
98
+ context "whole second precision" do
99
+ let(:parsable_string) { "Mar 17 2021 12:34:56 +00:00" }
100
+ it "is accurate to the second" do
101
+ expect(parsed_result.nsec).to eq(000_000_000)
102
+ expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56Z"))
103
+ end
104
+ end
105
+ context "millisecond sub-second precision" do
106
+ let(:parsable_string) { "Mar 17 2021 12:34:56.987" }
107
+ let(:format_string) { "%b %d %H:%M:%S.%3N" }
108
+ it "is accurate to the millisecond" do
109
+ expect(parsed_result.nsec).to eq(987_000_000)
110
+ expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987Z"))
111
+ end
112
+ end
113
+ context "microsecond sub-second precision" do
114
+ let(:parsable_string) { "Mar 17 2021 12:34:56.987654" }
115
+ let(:format_string) { "%b %d %H:%M:%S.%6N" }
116
+ it "is accurate to the microsecond" do
117
+ expect(parsed_result.nsec).to eq(987_654_000)
118
+ expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987654Z"))
119
+ end
120
+ end
121
+ context "nanosecond sub-second precision" do
122
+ let(:parsable_string) { "Mar 17 2021 12:34:56.987654321" }
123
+ let(:format_string) { "%b %d %H:%M:%S.%9N" }
124
+ it "is accurate to the nanosecond" do
125
+ expect(parsed_result.nsec).to eq(987_654_321)
126
+ expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987654321Z"))
127
+ end
128
+ end
129
+ end
130
+
131
+ context "parsing dates with no year specified" do
132
+ let(:time_of_parse) { fail(NotImplementedError) }
133
+ let(:format_to_parse) { "%b %d %H:%M:%S.%3N" }
134
+ let(:offset_days) { fail(NotImplementedError) }
135
+ let(:time_to_parse) { (time_of_parse + (offset_days * 86400)) }
136
+ let(:parsable_string) { time_to_parse.strftime(format_to_parse) }
137
+
138
+
139
+ let(:anchored_clock) do
140
+ instant = java.time.Instant.of_epoch_second(time_of_parse.to_i)
141
+ zone = java.time.ZoneId.system_default
142
+
143
+ java.time.Clock.fixed(instant, zone)
144
+ end
145
+
146
+ subject(:timestamp_normalizer) { described_class.new(clock: anchored_clock) }
147
+
148
+ let(:parsed_result) { timestamp_normalizer.normalize(parsable_string) }
149
+
150
+ context 'when parsing a date during late December' do
151
+ let(:time_of_parse) { Time.parse("2020-12-31T23:53:08.123456789Z") }
152
+ context 'and handling a date string from early january' do
153
+ let(:time_to_parse) { Time.parse("2021-01-01T00:00:08.123456789Z") }
154
+ it 'assumes that the date being parsed is in the very near future' do
155
+ expect(parsed_result.month).to eq(1)
156
+ expect(parsed_result.year).to eq(time_of_parse.year + 1)
157
+ end
158
+ end
159
+ context 'and handling a yearless date string from mid january' do
160
+ let(:time_to_parse) { Time.parse("2021-01-17T00:00:08.123456789Z") }
161
+ it 'assumes that the date being parsed is in the distant past' do
162
+ $stderr.puts(parsable_string)
163
+ expect(parsed_result.month).to eq(1)
164
+ expect(parsed_result.year).to eq(time_of_parse.year)
165
+ end
166
+ end
167
+ end
168
+
169
+ # As a smoke test to validate the guess-the-year feature when the provided CEF timestamp
170
+ # does not include the year, we iterate through a variety of dates that we want to parse,
171
+ # and with each of them we parse with a mock clock as if we were performing the parsing
172
+ # operation at a variety of date-times relative to the timestamp represented.
173
+ %w(
174
+ 2021-01-20T04:10:22.961Z
175
+ 2021-06-08T03:38:55.518Z
176
+ 2021-07-12T18:46:12.149Z
177
+ 2021-08-12T04:17:36.680Z
178
+ 2021-08-12T13:20:14.951Z
179
+ 2021-09-17T13:18:57.534Z
180
+ 2021-09-23T16:35:40.404Z
181
+ 2021-10-30T18:52:29.263Z
182
+ 2021-11-11T00:52:39.409Z
183
+ 2021-11-19T13:37:07.189Z
184
+ 2021-12-02T01:09:21.846Z
185
+ 2021-12-11T16:35:05.641Z
186
+ 2021-12-15T14:17:22.152Z
187
+ 2021-12-19T05:53:57.200Z
188
+ 2021-12-20T16:18:17.637Z
189
+ 2021-12-22T12:06:48.965Z
190
+ 2021-12-26T04:45:14.964Z
191
+ 2022-01-05T09:42:39.895Z
192
+ 2022-02-02T04:58:22.080Z
193
+ 2022-02-05T08:10:15.386Z
194
+ 2022-02-15T16:48:27.083Z
195
+ 2022-02-31T13:26:55.298Z
196
+ 2022-03-10T20:16:25.732Z
197
+ 2022-03-20T23:38:58.734Z
198
+ 2022-03-30T03:42:09.546Z
199
+ 2022-04-09T05:55:18.697Z
200
+ 2022-04-14T05:05:29.278Z
201
+ 2022-04-25T15:29:19.567Z
202
+ 2022-05-02T08:34:21.666Z
203
+ 2022-05-24T02:59:02.257Z
204
+ 2022-07-25T01:58:35.713Z
205
+ 2022-07-27T03:27:57.568Z
206
+ 2022-07-28T20:28:22.704Z
207
+ 2022-09-21T08:59:10.508Z
208
+ 2022-10-29T23:54:02.372Z
209
+ 2022-11-12T15:22:51.758Z
210
+ 2022-11-22T22:02:33.278Z
211
+ 2022-12-30T03:18:38.333Z
212
+ 2023-01-02T16:55:57.829Z
213
+ 2023-01-13T16:37:38.078Z
214
+ 2023-01-27T07:27:09.296Z
215
+ 2023-01-30T17:56:43.665Z
216
+ 2023-02-18T11:41:18.886Z
217
+ 2023-02-28T18:51:59.504Z
218
+ 2023-03-10T06:52:14.285Z
219
+ 2023-04-17T16:25:06.489Z
220
+ 2023-04-18T20:46:29.611Z
221
+ 2023-04-27T10:21:41.036Z
222
+ 2023-05-08T02:54:57.131Z
223
+ 2023-05-13T01:17:37.396Z
224
+ 2023-05-24T18:23:05.136Z
225
+ 2023-06-01T11:09:48.129Z
226
+ 2023-06-22T07:44:56.876Z
227
+ 2023-06-25T20:17:44.394Z
228
+ 2023-06-25T20:53:36.329Z
229
+ 2023-07-24T13:07:58.536Z
230
+ 2023-07-27T21:35:54.299Z
231
+ 2023-08-07T11:15:33.803Z
232
+ 2023-08-12T18:45:46.791Z
233
+ 2023-08-19T23:22:19.717Z
234
+ 2023-08-22T23:19:41.075Z
235
+ 2023-08-25T15:22:47.405Z
236
+ 2023-09-03T14:34:13.345Z
237
+ 2023-09-28T05:48:20.040Z
238
+ 2023-09-29T21:14:15.531Z
239
+ 2023-11-12T21:25:55.233Z
240
+ 2023-11-30T00:41:21.834Z
241
+ 2023-12-11T10:14:51.676Z
242
+ 2023-12-14T18:02:33.005Z
243
+ 2023-12-18T09:00:43.589Z
244
+ 2023-12-20T20:02:42.205Z
245
+ 2023-12-22T10:13:37.553Z
246
+ 2023-12-27T19:42:37.905Z
247
+ 2023-12-31T17:52:50.101Z
248
+ 2024-02-29T01:23:45.678Z
249
+ ).map {|ts| Time.parse(ts) }.each do |timestamp|
250
+ cef_parsable_timestamp = timestamp.strftime("%b %d %H:%M:%S.%3N Z")
251
+
252
+ context "when parsing the string `#{cef_parsable_timestamp}`" do
253
+
254
+ let(:expected_result) { timestamp }
255
+ let(:parsable_string) { cef_parsable_timestamp }
256
+
257
+ {
258
+ 'very recent past' => -30.789, # ~ 30 seconds ago
259
+ 'somewhat recent past' => -608976.678, # ~ 1 week days ago
260
+ 'distant past' => -29879991.916, # ~ 11-1/2 months days ago
261
+ 'near future' => 132295.719, # ~ 1.5 days from now
262
+ }.each do |desc, shift|
263
+ shifted_now = timestamp - shift
264
+ context "when that string could plausibly be in the #{desc} (NOW: #{shifted_now.iso8601(3)})" do
265
+ let(:time_of_parse) { shifted_now }
266
+ it "produces a time in the #{desc} (#{timestamp.iso8601(3)})" do
267
+ expect(parsed_result).to eq(expected_result)
268
+ end
269
+ end
270
+ end
271
+ end
272
+ end
273
+ end
274
+ end
data/spec/codecs/cef_spec.rb CHANGED
@@ -1,16 +1,19 @@
1
1
  # encoding: utf-8
2
+ require 'logstash/util'
2
3
  require "logstash/devutils/rspec/spec_helper"
3
4
  require "insist"
4
5
  require "logstash/codecs/cef"
5
6
  require "logstash/event"
6
7
  require "json"
7
8
 
9
+ require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
10
+
8
11
  describe LogStash::Codecs::CEF do
9
- subject do
12
+ subject(:codec) do
10
13
  next LogStash::Codecs::CEF.new
11
14
  end
12
15
 
13
- context "#encode" do
16
+ context "#encode", :ecs_compatibility_support do
14
17
  subject(:codec) { LogStash::Codecs::CEF.new }
15
18
 
16
19
  let(:results) { [] }
@@ -210,25 +213,92 @@ describe LogStash::Codecs::CEF do
210
213
  codec.encode(event)
211
214
  expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|foo=[0-9TZ.:-]+$/m)
212
215
  end
213
-
214
- it "should encode the CEF field names to their long versions" do
215
- # This is with the default value of "reverse_mapping" that is "false".
216
- codec.on_event{|data, newdata| results << newdata}
217
- codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
218
- event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
219
- codec.encode(event)
220
- expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=foobar applicationProtocol=foobar deviceCustomIPv6Address1=foobar deviceCustomIPv6Address1Label=foobar deviceCustomIPv6Address2=foobar deviceCustomIPv6Address2Label=foobar deviceCustomIPv6Address3=foobar deviceCustomIPv6Address3Label=foobar deviceCustomIPv6Address4=foobar deviceCustomIPv6Address4Label=foobar deviceEventCategory=foobar deviceCustomFloatingPoint1=foobar deviceCustomFloatingPoint1Label=foobar deviceCustomFloatingPoint2=foobar deviceCustomFloatingPoint2Label=foobar deviceCustomFloatingPoint3=foobar deviceCustomFloatingPoint3Label=foobar deviceCustomFloatingPoint4=foobar deviceCustomFloatingPoint4Label=foobar deviceCustomNumber1=foobar deviceCustomNumber1Label=foobar deviceCustomNumber2=foobar deviceCustomNumber2Label=foobar deviceCustomNumber3=foobar deviceCustomNumber3Label=foobar baseEventCount=foobar deviceCustomString1=foobar deviceCustomString1Label=foobar deviceCustomString2=foobar deviceCustomString2Label=foobar deviceCustomString3=foobar deviceCustomString3Label=foobar deviceCustomString4=foobar deviceCustomString4Label=foobar deviceCustomString5=foobar deviceCustomString5Label=foobar deviceCustomString6=foobar deviceCustomString6Label=foobar destinationHostName=foobar destinationMacAddress=foobar destinationNtDomain=foobar destinationProcessId=foobar destinationUserPrivileges=foobar destinationProcessName=foobar destinationPort=foobar destinationAddress=foobar destinationUserId=foobar destinationUserName=foobar deviceAddress=foobar deviceHostName=foobar deviceProcessId=foobar endTime=foobar fileName=foobar fileSize=foobar bytesIn=foobar message=foobar bytesOut=foobar eventOutcome=foobar transportProtocol=foobar requestUrl=foobar deviceReceiptTime=foobar sourceHostName=foobar sourceMacAddress=foobar sourceNtDomain=foobar sourceProcessId=foobar sourceUserPrivileges=foobar sourceProcessName=foobar sourcePort=foobar sourceAddress=foobar startTime=foobar sourceUserId=foobar sourceUserName=foobar agentHostName=foobar agentReceiptTime=foobar agentType=foobar agentId=foobar agentAddress=foobar agentVersion=foobar agentTimeZone=foobar destinationTimeZone=foobar sourceLongitude=foobar sourceLatitude=foobar destinationLongitude=foobar destinationLatitude=foobar categoryDeviceType=foobar managerReceiptTime=foobar agentMacAddress=foobar$/m)
221
- end
222
216
 
223
- context "with reverse_mapping set to true" do
224
- subject(:codec) { LogStash::Codecs::CEF.new("reverse_mapping" => true) }
217
+ ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
218
+ before(:each) do
219
+ allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
220
+ end
225
221
 
226
- it "should encode the CEF field names to their short versions" do
222
+ it "should encode the CEF field names to their long versions" do
223
+ # This is with the default value of "reverse_mapping" that is "false".
227
224
  codec.on_event{|data, newdata| results << newdata}
228
225
  codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
229
226
  event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
230
227
  codec.encode(event)
231
- expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=foobar app=foobar c6a1=foobar c6a1Label=foobar c6a2=foobar c6a2Label=foobar c6a3=foobar c6a3Label=foobar c6a4=foobar c6a4Label=foobar cat=foobar cfp1=foobar cfp1Label=foobar cfp2=foobar cfp2Label=foobar cfp3=foobar cfp3Label=foobar cfp4=foobar cfp4Label=foobar cn1=foobar cn1Label=foobar cn2=foobar cn2Label=foobar cn3=foobar cn3Label=foobar cnt=foobar cs1=foobar cs1Label=foobar cs2=foobar cs2Label=foobar cs3=foobar cs3Label=foobar cs4=foobar cs4Label=foobar cs5=foobar cs5Label=foobar cs6=foobar cs6Label=foobar dhost=foobar dmac=foobar dntdom=foobar dpid=foobar dpriv=foobar dproc=foobar dpt=foobar dst=foobar duid=foobar duser=foobar dvc=foobar dvchost=foobar dvcpid=foobar end=foobar fname=foobar fsize=foobar in=foobar msg=foobar out=foobar outcome=foobar proto=foobar request=foobar rt=foobar shost=foobar smac=foobar sntdom=foobar spid=foobar spriv=foobar sproc=foobar spt=foobar src=foobar start=foobar suid=foobar suser=foobar ahost=foobar art=foobar at=foobar aid=foobar agt=foobar av=foobar atz=foobar dtz=foobar slong=foobar slat=foobar dlong=foobar dlat=foobar catdt=foobar mrt=foobar amac=foobar$/m)
228
+ expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=foobar applicationProtocol=foobar deviceCustomIPv6Address1=foobar deviceCustomIPv6Address1Label=foobar deviceCustomIPv6Address2=foobar deviceCustomIPv6Address2Label=foobar deviceCustomIPv6Address3=foobar deviceCustomIPv6Address3Label=foobar deviceCustomIPv6Address4=foobar deviceCustomIPv6Address4Label=foobar deviceEventCategory=foobar deviceCustomFloatingPoint1=foobar deviceCustomFloatingPoint1Label=foobar deviceCustomFloatingPoint2=foobar deviceCustomFloatingPoint2Label=foobar deviceCustomFloatingPoint3=foobar deviceCustomFloatingPoint3Label=foobar deviceCustomFloatingPoint4=foobar deviceCustomFloatingPoint4Label=foobar deviceCustomNumber1=foobar deviceCustomNumber1Label=foobar deviceCustomNumber2=foobar deviceCustomNumber2Label=foobar deviceCustomNumber3=foobar deviceCustomNumber3Label=foobar baseEventCount=foobar deviceCustomString1=foobar deviceCustomString1Label=foobar deviceCustomString2=foobar deviceCustomString2Label=foobar deviceCustomString3=foobar deviceCustomString3Label=foobar deviceCustomString4=foobar deviceCustomString4Label=foobar deviceCustomString5=foobar deviceCustomString5Label=foobar deviceCustomString6=foobar deviceCustomString6Label=foobar destinationHostName=foobar destinationMacAddress=foobar destinationNtDomain=foobar destinationProcessId=foobar destinationUserPrivileges=foobar destinationProcessName=foobar destinationPort=foobar destinationAddress=foobar destinationUserId=foobar destinationUserName=foobar deviceAddress=foobar deviceHostName=foobar deviceProcessId=foobar endTime=foobar fileName=foobar fileSize=foobar bytesIn=foobar message=foobar bytesOut=foobar eventOutcome=foobar transportProtocol=foobar requestUrl=foobar deviceReceiptTime=foobar sourceHostName=foobar sourceMacAddress=foobar sourceNtDomain=foobar sourceProcessId=foobar sourceUserPrivileges=foobar sourceProcessName=foobar sourcePort=foobar sourceAddress=foobar startTime=foobar sourceUserId=foobar sourceUserName=foobar agentHostName=foobar agentReceiptTime=foobar agentType=foobar agentId=foobar agentAddress=foobar agentVersion=foobar agentTimeZone=foobar destinationTimeZone=foobar sourceLongitude=foobar sourceLatitude=foobar destinationLongitude=foobar destinationLatitude=foobar categoryDeviceType=foobar managerReceiptTime=foobar agentMacAddress=foobar$/m)
229
+ end
230
+
231
+ if ecs_select.active_mode != :disabled
232
+ let(:event_flat_hash) do
233
+ {
234
+ "[event][action]" => "floop", # deviceAction
235
+ "[network][protocol]" => "https", # applicationProtocol
236
+ "[cef][device_custom_ipv6_address_1][value]" => "4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9", # deviceCustomIPv6Address1
237
+ "[cef][device_custom_ipv6_address_1][label]" => "internal-interface", # deviceCustomIPv6Address1Label
238
+ "[observer][ip]" => "123.45.67.89", # deviceAddress
239
+ "[observer][hostname]" => "banana", # deviceHostName
240
+ "[user_agent][original]" => "'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='", # requestClientApplication
241
+ "[source][registered_domain]" => "monkey.see" # sourceDnsDomain
242
+ }
243
+ end
244
+
245
+ let(:event) do
246
+ event_flat_hash.each_with_object(LogStash::Event.new) do |(fr,v),memo|
247
+ memo.set(fr, v)
248
+ end
249
+ end
250
+
251
+ it 'encodes the ECS field names to their CEF name' do
252
+ codec.on_event{|data, newdata| results << newdata}
253
+ codec.fields = event_flat_hash.keys
254
+
255
+ codec.encode(event)
256
+
257
+ expect(results.first).to match(%r{^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=floop applicationProtocol=https deviceCustomIPv6Address1=4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9 deviceCustomIPv6Address1Label=internal-interface deviceAddress=123\.45\.67\.89 deviceHostName=banana requestClientApplication='Foo-Bar/2018\.1\.7; Email:user@example\.com; Guid:test\\=' sourceDnsDomain=monkey.see$}m)
258
+ end
259
+ end
260
+
261
+ context "with reverse_mapping set to true" do
262
+ subject(:codec) { LogStash::Codecs::CEF.new("reverse_mapping" => true) }
263
+
264
+ it "should encode the CEF field names to their short versions" do
265
+ codec.on_event{|data, newdata| results << newdata}
266
+ codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
267
+ event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
268
+ codec.encode(event)
269
+ expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=foobar app=foobar c6a1=foobar c6a1Label=foobar c6a2=foobar c6a2Label=foobar c6a3=foobar c6a3Label=foobar c6a4=foobar c6a4Label=foobar cat=foobar cfp1=foobar cfp1Label=foobar cfp2=foobar cfp2Label=foobar cfp3=foobar cfp3Label=foobar cfp4=foobar cfp4Label=foobar cn1=foobar cn1Label=foobar cn2=foobar cn2Label=foobar cn3=foobar cn3Label=foobar cnt=foobar cs1=foobar cs1Label=foobar cs2=foobar cs2Label=foobar cs3=foobar cs3Label=foobar cs4=foobar cs4Label=foobar cs5=foobar cs5Label=foobar cs6=foobar cs6Label=foobar dhost=foobar dmac=foobar dntdom=foobar dpid=foobar dpriv=foobar dproc=foobar dpt=foobar dst=foobar duid=foobar duser=foobar dvc=foobar dvchost=foobar dvcpid=foobar end=foobar fname=foobar fsize=foobar in=foobar msg=foobar out=foobar outcome=foobar proto=foobar request=foobar rt=foobar shost=foobar smac=foobar sntdom=foobar spid=foobar spriv=foobar sproc=foobar spt=foobar src=foobar start=foobar suid=foobar suser=foobar ahost=foobar art=foobar at=foobar aid=foobar agt=foobar av=foobar atz=foobar dtz=foobar slong=foobar slat=foobar dlong=foobar dlat=foobar catdt=foobar mrt=foobar amac=foobar$/m)
270
+ end
271
+
272
+ if ecs_select.active_mode != :disabled
273
+ let(:event_flat_hash) do
274
+ {
275
+ "[event][action]" => "floop", # act
276
+ "[network][protocol]" => "https", # app
277
+ "[cef][device_custom_ipv6_address_1][value]" => "4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9", # c6a1
278
+ "[cef][device_custom_ipv6_address_1][label]" => "internal-interface", # c6a1Label
279
+ "[observer][ip]" => "123.45.67.89", # dvc
280
+ "[observer][hostname]" => "banana", # dvchost
281
+ "[user_agent][original]" => "'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='",
282
+ "[source][registered_domain]" => "monkey.see" # sourceDnsDomain
283
+ }
284
+ end
285
+
286
+ let(:event) do
287
+ event_flat_hash.each_with_object(LogStash::Event.new) do |(fr,v),memo|
288
+ memo.set(fr, v)
289
+ end
290
+ end
291
+
292
+
293
+ it 'encodes the ECS field names to their CEF keys' do
294
+ codec.on_event{|data, newdata| results << newdata}
295
+ codec.fields = event_flat_hash.keys
296
+
297
+ codec.encode(event)
298
+
299
+ expect(results.first).to match(%r{^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=floop app=https c6a1=4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9 c6a1Label=internal-interface dvc=123\.45\.67\.89 dvchost=banana requestClientApplication='Foo-Bar/2018\.1\.7; Email:user@example\.com; Guid:test\\=' sourceDnsDomain=monkey.see$}m)
300
+ end
301
+ end
232
302
  end
233
303
  end
234
304
  end
@@ -306,11 +376,21 @@ describe LogStash::Codecs::CEF do
306
376
  end
307
377
  end
308
378
 
309
- context "#decode" do
310
- let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
311
-
379
+ module DecodeHelpers
312
380
  def validate(e)
313
381
  insist { e.is_a?(LogStash::Event) }
382
+ send("validate_ecs_#{ecs_compatibility}", e)
383
+ end
384
+
385
+ def validate_ecs_v1(e)
386
+ insist { e.get('[cef][version]') } == "0"
387
+ insist { e.get('[observer][version]') } == "1.0"
388
+ insist { e.get('[event][code]') } == "100"
389
+ insist { e.get('[cef][name]') } == "trojan successfully stopped"
390
+ insist { e.get('[event][severity]') } == "10"
391
+ end
392
+
393
+ def validate_ecs_disabled(e)
314
394
  insist { e.get('cefVersion') } == "0"
315
395
  insist { e.get('deviceVersion') } == "1.0"
316
396
  insist { e.get('deviceEventClassId') } == "100"
@@ -334,7 +414,11 @@ describe LogStash::Codecs::CEF do
334
414
  fail("Expected one event, got #{events.size} events: #{events.inspect}") unless events.size == 1
335
415
  event = events.first
336
416
 
337
- yield event if block_given?
417
+ if block_given?
418
+ aggregate_failures('decode one') do
419
+ yield event
420
+ end
421
+ end
338
422
 
339
423
  event
340
424
  end
@@ -360,369 +444,479 @@ describe LogStash::Codecs::CEF do
360
444
 
361
445
  events
362
446
  end
447
+ end
363
448
 
364
- context "with delimiter set" do
365
- # '\r\n' in single quotes to simulate the real input from a config
366
- # containing \r\n as 4-character sequence in the config:
367
- #
368
- # delimiter => "\r\n"
369
- #
370
- # Related: https://github.com/elastic/logstash/issues/1645
371
- subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
449
+ context "#decode", :ecs_compatibility_support do
450
+ ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
451
+ before(:each) do
452
+ allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
453
+ end
372
454
 
373
- it "should parse on the delimiter " do
374
- do_decode(subject,message) do |e|
375
- raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
376
- end
455
+ let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
377
456
 
378
- decode_one(subject, "\r\n") do |e|
379
- validate(e)
380
- insist { e.get("deviceVendor") } == "security"
381
- insist { e.get("deviceProduct") } == "threatmanager"
457
+ include DecodeHelpers
458
+
459
+ context "with delimiter set" do
460
+ # '\r\n' in single quotes to simulate the real input from a config
461
+ # containing \r\n as 4-character sequence in the config:
462
+ #
463
+ # delimiter => "\r\n"
464
+ #
465
+ # Related: https://github.com/elastic/logstash/issues/1645
466
+ subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
467
+
468
+ it "should parse on the delimiter " do
469
+ do_decode(subject,message) do |e|
470
+ raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
471
+ end
472
+
473
+ decode_one(subject, "\r\n") do |e|
474
+ validate(e)
475
+ insist { e.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "security"
476
+ insist { e.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "threatmanager"
477
+ end
382
478
  end
383
479
  end
384
- end
385
480
 
386
- context 'when a CEF header ends with a pair of properly-escaped backslashes' do
387
- let(:backslash) { '\\' }
388
- let(:pipe) { '|' }
389
- let(:message) { "CEF:0|security|threatmanager|1.0|100|double backslash" +
390
- backslash + backslash + # escaped backslash
391
- backslash + backslash + # escaped backslash
392
- "|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
481
+ context 'when a CEF header ends with a pair of properly-escaped backslashes' do
482
+ let(:backslash) { '\\' }
483
+ let(:pipe) { '|' }
484
+ let(:message) { "CEF:0|security|threatmanager|1.0|100|double backslash" +
485
+ backslash + backslash + # escaped backslash
486
+ backslash + backslash + # escaped backslash
487
+ "|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
393
488
 
394
- it 'should include the backslashes unescaped' do
395
- event = decode_one(subject, message)
489
+ it 'should include the backslashes unescaped' do
490
+ event = decode_one(subject, message)
396
491
 
397
- expect(event.get('name')).to eq('double backslash' + backslash + backslash )
398
- expect(event.get('severity')).to eq('10') # ensure we didn't consume the separator
492
+ expect(event.get(ecs_select[disabled:'name', v1:'[cef][name]'])).to eq('double backslash' + backslash + backslash )
493
+ expect(event.get(ecs_select[disabled:'severity',v1:'[event][severity]'])).to eq('10') # ensure we didn't consume the separator
494
+ end
399
495
  end
400
- end
401
496
 
402
- it "should parse the cef headers" do
403
- decode_one(subject, message) do |e|
404
- validate(e)
405
- insist { e.get("deviceVendor") } == "security"
406
- insist { e.get("deviceProduct") } == "threatmanager"
497
+ it "should parse the cef headers" do
498
+ decode_one(subject, message) do |e|
499
+ validate(e)
500
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == "security"
501
+ insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == "threatmanager"
502
+ end
407
503
  end
408
- end
409
504
 
410
- it "should parse the cef body" do
411
- decode_one(subject, message) do |e|
412
- insist { e.get("sourceAddress")} == "10.0.0.192"
413
- insist { e.get("destinationAddress") } == "12.121.122.82"
414
- insist { e.get("sourcePort") } == "1232"
505
+ it "should parse the cef body" do
506
+ decode_one(subject, message) do |e|
507
+ insist { e.get(ecs_select[disabled:"sourceAddress", v1:"[source][ip]"])} == "10.0.0.192"
508
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
509
+ insist { e.get(ecs_select[disabled:"sourcePort", v1:"[source][port]"]) } == "1232"
510
+ end
415
511
  end
416
- end
417
512
 
418
- let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
419
- it "should be OK with missing CEF headers (multiple pipes in sequence)" do
420
- decode_one(subject, missing_headers) do |e|
421
- validate(e)
422
- insist { e.get("deviceVendor") } == ""
423
- insist { e.get("deviceProduct") } == ""
513
+ let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
514
+ it "should be OK with missing CEF headers (multiple pipes in sequence)" do
515
+ decode_one(subject, missing_headers) do |e|
516
+ validate(e)
517
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == ""
518
+ insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == ""
519
+ end
424
520
  end
425
- end
426
521
 
427
- let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
428
- it "should strip leading whitespace from the message" do
429
- decode_one(subject, leading_whitespace) do |e|
430
- validate(e)
522
+ let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
523
+ it "should strip leading whitespace from the message" do
524
+ decode_one(subject, leading_whitespace) do |e|
525
+ validate(e)
526
+ end
431
527
  end
432
- end
433
528
 
434
- let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
435
- it "should be OK with escaped pipes in the message" do
436
- decode_one(subject, escaped_pipes) do |e|
437
- insist { e.get("moo") } == 'this\|has an escaped pipe'
529
+ let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
530
+ it "should be OK with escaped pipes in the message" do
531
+ decode_one(subject, escaped_pipes) do |e|
532
+ insist { e.get("moo") } == 'this\|has an escaped pipe'
533
+ end
438
534
  end
439
- end
440
535
 
441
- let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
442
- it "should be OK with not escaped pipes in the message" do
443
- decode_one(subject, pipes_in_message) do |e|
444
- insist { e.get("moo") } == 'this|has an pipe'
536
+ let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
537
+ it "should be OK with not escaped pipes in the message" do
538
+ decode_one(subject, pipes_in_message) do |e|
539
+ insist { e.get("moo") } == 'this|has an pipe'
540
+ end
445
541
  end
446
- end
447
542
 
448
- # while we may see these in practice, equals MUST be escaped in the extensions per the spec.
449
- let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
450
- it "should be OK with equal in the message" do
451
- decode_one(subject, equal_in_message) do |e|
452
- insist { e.get("moo") } == 'this =has = equals='
543
+ # while we may see these in practice, equals MUST be escaped in the extensions per the spec.
544
+ let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
545
+ it "should be OK with equal in the message" do
546
+ decode_one(subject, equal_in_message) do |e|
547
+ insist { e.get("moo") } == 'this =has = equals='
548
+ end
453
549
  end
454
- end
455
550
 
456
- let(:malformed_unescaped_equals_in_extension_value) { %q{CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar} }
457
- it 'should split correctly' do
458
- decode_one(subject, malformed_unescaped_equals_in_extension_value) do |event|
459
- expect(event.get('cefVersion')).to eq('0')
460
- expect(event.get('deviceVendor')).to eq('FooBar')
461
- expect(event.get('deviceProduct')).to eq('Web Gateway')
462
- expect(event.get('deviceVersion')).to eq('1.2.3.45.67')
463
- expect(event.get('deviceEventClassId')).to eq('200')
464
- expect(event.get('name')).to eq('Success')
465
- expect(event.get('severity')).to eq('2')
466
-
467
- # extension key/value pairs
468
- expect(event.get('deviceReceiptTime')).to eq('Sep 07 2018 14:50:39')
469
- expect(event.get('deviceEventCategory')).to eq('Access Log')
470
- expect(event.get('deviceVersion')).to eq('1.2.3.45.67')
471
- expect(event.get('destinationAddress')).to eq('1.1.1.1')
472
- expect(event.get('destinationHostName')).to eq('foo.example.com')
473
- expect(event.get('sourceUserName')).to eq('redacted')
474
- expect(event.get('sourceAddress')).to eq('2.2.2.2')
475
- expect(event.get('requestMethod')).to eq('POST')
476
- expect(event.get('requestUrl')).to eq(%q{'https://foo.example.com/bar/bingo/1'})
477
- # Although the value for `requestClientApplication` contains an illegal unquoted equals sign, the sequence
478
- # preceeding the unescaped-equals isn't shaped like a key, so we allow it to be a part of the value.
479
- expect(event.get('requestClientApplication')).to eq(%q{'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='})
480
- expect(event.get('deviceCustomString1Label')).to eq('Foo Bar')
481
- expect(event.get('deviceCustomString1')).to eq('')
551
+ context "zoneless deviceReceiptTime(rt) when deviceTimeZone(dtz) is provided" do
552
+ let(:cef_formatted_timestamp) { 'Jul 19 2017 10:50:21.127' }
553
+ let(:zone_name) { 'Europe/Moscow' }
554
+
555
+ let(:utc_timestamp) { Time.iso8601("2017-07-19T07:50:21.127Z") } # In summer of 2017, Europe/Moscow was UTC+03:00
556
+
557
+ let(:destination_time_zoned) { %Q{CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|Very-High| eventId=1 msg=Worm successfully stopped art=1500464384997 deviceSeverity=10 rt=#{cef_formatted_timestamp} src=10.0.0.1 sourceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 10.0.0.0-10.255.255.255 spt=1232 dst=2.1.2.2 destinationZoneURI=/All Zones/ArcSight System/Public Address Space Zones/RIPE NCC/2.0.0.0-2.255.255.255 (RIPE NCC) ahost=connector.rhel72 agt=192.168.231.129 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-51-8A-84 av=7.6.0.8009.0 atz=Europe/Lisbon at=syslog_file dvchost=client1 dtz=#{zone_name} _cefVer=0.1 aid=3UBajWl0BABCABBzZSlmUdw==} }
558
+
559
+ if ecs_select.active_mode == :disabled
560
+ it 'persists deviceReceiptTime and deviceTimeZone verbatim' do
561
+ decode_one(subject, destination_time_zoned) do |event|
562
+ expect(event.get('deviceReceiptTime')).to eq("Jul 19 2017 10:50:21.127")
563
+ expect(event.get('deviceTimeZone')).to eq('Europe/Moscow')
564
+ end
565
+ end
566
+ else
567
+ it 'sets the @timestamp using the value in `rt` combined with the offset provided by `dtz`' do
568
+ decode_one(subject, destination_time_zoned) do |event|
569
+ expected_time = LogStash::Timestamp.new(utc_timestamp)
570
+ expect(event.get('[@timestamp]').to_s).to eq(expected_time.to_s)
571
+ expect(event.get('[event][timezone]')).to eq(zone_name)
572
+ end
573
+ end
574
+ end
575
+ end
576
+
577
+ let(:malformed_unescaped_equals_in_extension_value) { %q{CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar} }
578
+ it 'should split correctly' do
579
+ decode_one(subject, malformed_unescaped_equals_in_extension_value) do |event|
580
+ expect(event.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"])).to eq('0')
581
+ expect(event.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"])).to eq('FooBar')
582
+ expect(event.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"])).to eq('Web Gateway')
583
+ expect(event.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"])).to eq('1.2.3.45.67')
584
+ expect(event.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"])).to eq('200')
585
+ expect(event.get(ecs_select[disabled:"name", v1:"[cef][name]"])).to eq('Success')
586
+ expect(event.get(ecs_select[disabled:"severity", v1:"[event][severity]"])).to eq('2')
587
+
588
+ # extension key/value pairs
589
+ if ecs_compatibility == :disabled
590
+ expect(event.get('deviceReceiptTime')).to eq('Sep 07 2018 14:50:39')
591
+ else
592
+ expected_time = LogStash::Timestamp.new(Time.parse('Sep 07 2018 14:50:39')).to_s
593
+ expect(event.get('[@timestamp]').to_s).to eq(expected_time)
594
+ end
595
+ expect(event.get(ecs_select[disabled:'deviceEventCategory', v1:'[cef][category]'])).to eq('Access Log')
596
+ expect(event.get(ecs_select[disabled:'deviceVersion', v1:'[observer][version]'])).to eq('1.2.3.45.67')
597
+ expect(event.get(ecs_select[disabled:'destinationAddress', v1:'[destination][ip]'])).to eq('1.1.1.1')
598
+ expect(event.get(ecs_select[disabled:'destinationHostName', v1:'[destination][domain]'])).to eq('foo.example.com')
599
+ expect(event.get(ecs_select[disabled:'sourceUserName', v1:'[source][user][name]'])).to eq('redacted')
600
+ expect(event.get(ecs_select[disabled:'sourceAddress', v1:'[source][ip]'])).to eq('2.2.2.2')
601
+ expect(event.get(ecs_select[disabled:'requestMethod', v1:'[http][request][method]'])).to eq('POST')
602
+ expect(event.get(ecs_select[disabled:'requestUrl', v1:'[url][original]'])).to eq(%q{'https://foo.example.com/bar/bingo/1'})
603
+ # Although the value for `requestClientApplication` contains an illegal unquoted equals sign, the sequence
604
+ # preceeding the unescaped-equals isn't shaped like a key, so we allow it to be a part of the value.
605
+ expect(event.get(ecs_select[disabled:'requestClientApplication',v1:'[user_agent][original]'])).to eq(%q{'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='})
606
+ expect(event.get(ecs_select[disabled:'deviceCustomString1Label',v1:'[cef][device_custom_string_1][label]'])).to eq('Foo Bar')
607
+ expect(event.get(ecs_select[disabled:'deviceCustomString1', v1:'[cef][device_custom_string_1][value]'])).to eq('')
608
+ end
482
609
  end
483
- end
484
610
 
485
- context('escaped-equals and unescaped-spaces in the extension values') do
486
- let(:query_string) { 'key1=value1&key2=value3 aa.bc&key3=value4'}
487
- let(:escaped_query_string) { query_string.gsub('=','\\=') }
488
- let(:cef_message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|go=start now query_string=#{escaped_query_string} final=done" }
611
+ context('escaped-equals and unescaped-spaces in the extension values') do
612
+ let(:query_string) { 'key1=value1&key2=value3 aa.bc&key3=value4'}
613
+ let(:escaped_query_string) { query_string.gsub('=','\\=') }
614
+ let(:cef_message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|go=start now query_string=#{escaped_query_string} final=done" }
489
615
 
490
- it 'captures the extension values correctly' do
491
- event = decode_one(subject, cef_message)
616
+ it 'captures the extension values correctly' do
617
+ event = decode_one(subject, cef_message)
492
618
 
493
- expect(event.get('go')).to eq('start now')
494
- expect(event.get('query_string')).to eq(query_string)
495
- expect(event.get('final')).to eq('done')
619
+ expect(event.get('go')).to eq('start now')
620
+ expect(event.get('query_string')).to eq(query_string)
621
+ expect(event.get('final')).to eq('done')
622
+ end
496
623
  end
497
- end
498
624
 
499
- let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
500
- it "should be OK with escaped backslash in the headers" do
501
- decode_one(subject, escaped_backslash_in_header) do |e|
502
- insist { e.get("cefVersion") } == '0'
503
- insist { e.get("deviceVendor") } == 'secu\\rity'
504
- insist { e.get("deviceProduct") } == 'threat\\manager'
505
- insist { e.get("deviceVersion") } == '1.\\0'
506
- insist { e.get("deviceEventClassId") } == '10\\0'
507
- insist { e.get("name") } == 'tro\\jan successfully stopped'
508
- insist { e.get("severity") } == '\\10'
625
+ let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
626
+ it "should be OK with escaped backslash in the headers" do
627
+ decode_one(subject, escaped_backslash_in_header) do |e|
628
+ insist { e.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"]) } == '0'
629
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'secu\\rity'
630
+ insist { e.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"]) } == 'threat\\manager'
631
+ insist { e.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"]) } == '1.\\0'
632
+ insist { e.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"]) } == '10\\0'
633
+ insist { e.get(ecs_select[disabled:"name", v1:"[cef][name]"]) } == 'tro\\jan successfully stopped'
634
+ insist { e.get(ecs_select[disabled:"severity", v1:"[event][severity]"]) } == '\\10'
635
+ end
509
636
  end
510
- end
511
637
 
512
- let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
513
- it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
514
- decode_one(subject, escaped_backslash_in_header_edge_case) do |e|
515
- validate(e)
516
- insist { e.get("deviceVendor") } == 'security\\|'
517
- insist { e.get("deviceProduct") } == 'threatmanager\\'
638
+ let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
639
+ it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
640
+ decode_one(subject, escaped_backslash_in_header_edge_case) do |e|
641
+ validate(e)
642
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'security\\|'
643
+ insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == 'threatmanager\\'
644
+ end
518
645
  end
519
- end
520
646
 
521
- let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
522
- it "should be OK with escaped pipes in the headers" do
523
- decode_one(subject, escaped_pipes_in_header) do |e|
524
- insist { e.get("cefVersion") } == '0'
525
- insist { e.get("deviceVendor") } == 'secu|rity'
526
- insist { e.get("deviceProduct") } == 'threatmanager|'
527
- insist { e.get("deviceVersion") } == '1.|0'
528
- insist { e.get("deviceEventClassId") } == '10|0'
529
- insist { e.get("name") } == 'tro|jan successfully stopped'
530
- insist { e.get("severity") } == '|10'
647
+ let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
648
+ it "should be OK with escaped pipes in the headers" do
649
+ decode_one(subject, escaped_pipes_in_header) do |e|
650
+ insist { e.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"]) } == '0'
651
+ insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'secu|rity'
652
+ insist { e.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"]) } == 'threatmanager|'
653
+ insist { e.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"]) } == '1.|0'
654
+ insist { e.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"]) } == '10|0'
655
+ insist { e.get(ecs_select[disabled:"name", v1:"[cef][name]"]) } == 'tro|jan successfully stopped'
656
+ insist { e.get(ecs_select[disabled:"severity", v1:"[event][severity]"]) } == '|10'
657
+ end
531
658
  end
532
- end
533
659
 
534
- let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
535
- it "should be OK with backslashs in the message" do
536
- decode_one(subject, backslash_in_message) do |e|
537
- insist { e.get("moo") } == 'this \\has \\ backslashs\\'
660
+ let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
661
+ it "should be OK with backslashs in the message" do
662
+ decode_one(subject, backslash_in_message) do |e|
663
+ insist { e.get("moo") } == 'this \\has \\ backslashs\\'
664
+ end
538
665
  end
539
- end
540
666
 
541
- let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
542
- it "should be OK with equal in the headers" do
543
- decode_one(subject, equal_in_header) do |e|
544
- validate(e)
545
- insist { e.get("deviceProduct") } == "threatmanager=equal"
667
+ let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
668
+ it "should be OK with equal in the headers" do
669
+ decode_one(subject, equal_in_header) do |e|
670
+ validate(e)
671
+ insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == "threatmanager=equal"
672
+ end
546
673
  end
547
- end
548
674
 
549
- let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232'}
550
- it "should be OK to have one or more spaces between keys" do
551
- decode_one(subject, spaces_in_between_keys) do |e|
552
- validate(e)
553
- insist { e.get("sourceAddress") } == "10.0.0.192"
554
- insist { e.get("destinationAddress") } == "12.121.122.82"
555
- insist { e.get("sourcePort") } == "1232"
675
+ let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232'}
676
+ it "should be OK to have one or more spaces between keys" do
677
+ decode_one(subject, spaces_in_between_keys) do |e|
678
+ validate(e)
679
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
680
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
681
+ insist { e.get(ecs_select[disabled:"sourcePort",v1:"[source][port]"]) } == "1232"
682
+ end
556
683
  end
557
- end
558
684
 
559
- let (:dots_in_keys) {'CEF:0|Vendor|Device|Version|13|my message|5|dvchost=loghost cat=traffic deviceSeverity=notice ad.nn=TEST src=192.168.0.1 destinationPort=53'}
560
- it "should be OK with dots in keys" do
561
- decode_one(subject, dots_in_keys) do |e|
562
- insist { e.get("deviceHostName") } == "loghost"
563
- insist { e.get("ad.nn") } == 'TEST'
564
- insist { e.get("sourceAddress") } == '192.168.0.1'
565
- insist { e.get("destinationPort") } == '53'
685
+ let (:dots_in_keys) {'CEF:0|Vendor|Device|Version|13|my message|5|dvchost=loghost cat=traffic deviceSeverity=notice ad.nn=TEST src=192.168.0.1 destinationPort=53'}
686
+ it "should be OK with dots in keys" do
687
+ decode_one(subject, dots_in_keys) do |e|
688
+ insist { e.get(ecs_select[disabled:"deviceHostName",v1:"[observer][hostname]"]) } == "loghost"
689
+ insist { e.get("ad.nn") } == 'TEST'
690
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == '192.168.0.1'
691
+ insist { e.get(ecs_select[disabled:"destinationPort",v1:"[destination][port]"]) } == '53'
692
+ end
566
693
  end
567
- end
568
694
 
569
- let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 dproc=InternetExplorer x.x.x.x'}
570
- it "should be OK to have one or more spaces in values" do
571
- decode_one(subject, allow_spaces_in_values) do |e|
572
- validate(e)
573
- insist { e.get("sourceAddress") } == "10.0.0.192"
574
- insist { e.get("destinationAddress") } == "12.121.122.82"
575
- insist { e.get("sourcePort") } == "1232"
576
- insist { e.get("destinationProcessName") } == "InternetExplorer x.x.x.x"
695
+ let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 dproc=InternetExplorer x.x.x.x'}
696
+ it "should be OK to have one or more spaces in values" do
697
+ decode_one(subject, allow_spaces_in_values) do |e|
698
+ validate(e)
699
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
700
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
701
+ insist { e.get(ecs_select[disabled:"sourcePort",v1:"[source][port]"]) } == "1232"
702
+ insist { e.get(ecs_select[disabled:"destinationProcessName",v1:"[destination][process][name]"]) } == "InternetExplorer x.x.x.x"
703
+ end
577
704
  end
578
- end
579
705
 
580
- let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
581
- it "should keep ad.fields" do
582
- decode_one(subject, preserve_additional_fields_with_dot_notations) do |e|
583
- validate(e)
584
- insist { e.get("sourceAddress") } == "10.0.0.192"
585
- insist { e.get("destinationAddress") } == "12.121.122.82"
586
- insist { e.get("[ad.field][0]") } == "field0"
587
- insist { e.get("[ad.name][1]") } == "new_name"
588
- insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
589
- insist { e.get('ad.Error_,Code') } == "3221225578"
590
- insist { e.get("additional.dotfieldName") } == "new_value"
706
+ let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
707
+ it "should keep ad.fields" do
708
+ decode_one(subject, preserve_additional_fields_with_dot_notations) do |e|
709
+ validate(e)
710
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
711
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
712
+ insist { e.get("[ad.field][0]") } == "field0"
713
+ insist { e.get("[ad.name][1]") } == "new_name"
714
+ insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
715
+ insist { e.get('ad.Error_,Code') } == "3221225578"
716
+ insist { e.get("additional.dotfieldName") } == "new_value"
717
+ end
591
718
  end
592
- end
593
719
 
594
- let(:preserve_complex_multiple_dot_notation_in_extension_fields) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name' }
595
- it "should keep ad.fields" do
596
- decode_one(subject, preserve_complex_multiple_dot_notation_in_extension_fields) do |e|
597
- validate(e)
598
- insist { e.get("sourceAddress") } == "10.0.0.192"
599
- insist { e.get("destinationAddress") } == "12.121.122.82"
600
- insist { e.get("[ad.field][0]") } == "field0"
601
- insist { e.get("[ad.foo.name][1]") } == "new_name"
602
- insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
603
- insist { e.get('ad.Error_,Code') } == "3221225578"
604
- insist { e.get("additional.dotfieldName") } == "new_value"
720
+ let(:preserve_complex_multiple_dot_notation_in_extension_fields) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name' }
721
+ it "should keep ad.fields" do
722
+ decode_one(subject, preserve_complex_multiple_dot_notation_in_extension_fields) do |e|
723
+ validate(e)
724
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
725
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
726
+ insist { e.get("[ad.field][0]") } == "field0"
727
+ insist { e.get("[ad.foo.name][1]") } == "new_name"
728
+ insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
729
+ insist { e.get('ad.Error_,Code') } == "3221225578"
730
+ insist { e.get("additional.dotfieldName") } == "new_value"
731
+ end
605
732
  end
606
- end
607
733
 
608
- let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
609
- it "should correctly parse random values even with additional fields in message" do
610
- decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
611
- validate(e)
612
- insist { e.get("sourceAddress") } == "10.0.0.192"
613
- insist { e.get("destinationAddress") } == "12.121.122.82"
614
- insist { e.get("[ad.field][0]") } == "field0"
615
- insist { e.get("[ad.name][1]") } == "new_name"
616
- insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
617
- insist { e.get("ad.Error_,Code") } == "3221225578"
618
- insist { e.get("additional.dotfieldName") } == "new_value"
619
- insist { e.get("deviceCustomString4") } == "401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34"
734
+ let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
735
+ it "should correctly parse random values even with additional fields in message" do
736
+ decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
737
+ validate(e)
738
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
739
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
740
+ insist { e.get("[ad.field][0]") } == "field0"
741
+ insist { e.get("[ad.name][1]") } == "new_name"
742
+ insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
743
+ insist { e.get("ad.Error_,Code") } == "3221225578"
744
+ insist { e.get("additional.dotfieldName") } == "new_value"
745
+ insist { e.get(ecs_select[disabled:"deviceCustomString4",v1:"[cef][device_custom_string_4][value]"]) } == "401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34"
746
+ end
747
+ end
748
+
749
+ let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
750
+ it "should preserve unmatched key mappings" do
751
+ decode_one(subject, preserve_unmatched_key_mappings) do |e|
752
+ validate(e)
753
+ insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
754
+ insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
755
+ insist { e.get("new_key_by_device") } == "new_values here"
756
+ end
620
757
  end
621
- end
622
758
 
623
- let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
624
- it "should preserve unmatched key mappings" do
625
- decode_one(subject, preserve_unmatched_key_mappings) do |e|
626
- validate(e)
627
- insist { e.get("sourceAddress") } == "10.0.0.192"
628
- insist { e.get("destinationAddress") } == "12.121.122.82"
629
- insist { e.get("new_key_by_device") } == "new_values here"
759
+ let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
760
+ it "should translate most known abbreviated CEF field names" do
761
+ decode_one(subject, translate_abbreviated_cef_fields) do |e|
762
+ validate(e)
763
+ insist { e.get(ecs_select[disabled:"sourceAddress", v1:"[source][ip]"]) } == "10.0.0.192"
764
+ insist { e.get(ecs_select[disabled:"destinationAddress", v1:"[destination][ip]"]) } == "12.121.122.82"
765
+ insist { e.get(ecs_select[disabled:"transportProtocol", v1:"[network][transport]"]) } == "TCP"
766
+ insist { e.get(ecs_select[disabled:"sourceHostName", v1:"[source][domain]"]) } == "source.host.name"
767
+ insist { e.get(ecs_select[disabled:"destinationHostName",v1:"[destination][domain]"]) } == "destination.host.name"
768
+ insist { e.get(ecs_select[disabled:"sourcePort", v1:"[source][port]"]) } == "11024"
769
+ insist { e.get(ecs_select[disabled:"destinationPort", v1:"[destination][port]"]) } == "9200"
770
+ insist { e.get(ecs_select[disabled:"eventOutcome", v1:"[event][outcome]"]) } == "Success"
771
+ insist { e.get(ecs_select[disabled:"agentMacAddress", v1:"[agent][mac]"])} == "00:80:48:1c:24:91"
772
+ end
630
773
  end
631
- end
632
774
 
633
- let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
634
- it "should translate most known abbreviated CEF field names" do
635
- decode_one(subject, translate_abbreviated_cef_fields) do |e|
636
- validate(e)
637
- insist { e.get("sourceAddress") } == "10.0.0.192"
638
- insist { e.get("destinationAddress") } == "12.121.122.82"
639
- insist { e.get("transportProtocol") } == "TCP"
640
- insist { e.get("sourceHostName") } == "source.host.name"
641
- insist { e.get("destinationHostName") } == "destination.host.name"
642
- insist { e.get("sourcePort") } == "11024"
643
- insist { e.get("destinationPort") } == "9200"
644
- insist { e.get("eventOutcome") } == "Success"
645
- insist { e.get("agentMacAddress")} == "00:80:48:1c:24:91"
775
+ let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
776
+ it "Should detect headers before CEF starts" do
777
+ decode_one(subject, syslog) do |e|
778
+ validate(e)
779
+ insist { e.get(ecs_select[disabled:'syslog',v1:'[log][syslog][header]']) } == 'Syslogdate Sysloghost'
780
+ end
781
+ end
782
+
783
+ let(:log_with_fileHash) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|fileHash=1bad1dea" }
784
+ it 'decodes fileHash to [file][hash]' do
785
+ decode_one(subject, log_with_fileHash) do |e|
786
+ validate(e)
787
+ insist { e.get(ecs_select[disabled:"fileHash", v1:"[file][hash]"]) } == "1bad1dea"
788
+ end
646
789
  end
647
- end
648
790
 
649
- let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
650
- it "Should detect headers before CEF starts" do
651
- decode_one(subject, syslog) do |e|
652
- validate(e)
653
- insist { e.get('syslog') } == 'Syslogdate Sysloghost'
791
+ let(:log_with_custom_typed_fields) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|cfp15=3.1415926 cfp15Label=pi c6a12=::1 c6a12Label=localhost cn7=8191 cn7Label=mersenne cs4=silly cs4Label=theory" }
792
+ it 'decodes to mapped numbered fields' do
793
+ decode_one(subject, log_with_custom_typed_fields) do |e|
794
+ validate(e)
795
+ insist { e.get(ecs_select[disabled: "deviceCustomFloatingPoint15", v1: "[cef][device_custom_floating_point_15][value]"]) } == "3.1415926"
796
+ insist { e.get(ecs_select[disabled: "deviceCustomFloatingPoint15Label", v1: "[cef][device_custom_floating_point_15][label]"]) } == "pi"
797
+ insist { e.get(ecs_select[disabled: "deviceCustomIPv6Address12", v1: "[cef][device_custom_ipv6_address_12][value]"]) } == "::1"
798
+ insist { e.get(ecs_select[disabled: "deviceCustomIPv6Address12Label", v1: "[cef][device_custom_ipv6_address_12][label]"]) } == "localhost"
799
+ insist { e.get(ecs_select[disabled: "deviceCustomNumber7", v1: "[cef][device_custom_number_7][value]"]) } == "8191"
800
+ insist { e.get(ecs_select[disabled: "deviceCustomNumber7Label", v1: "[cef][device_custom_number_7][label]"]) } == "mersenne"
801
+ insist { e.get(ecs_select[disabled: "deviceCustomString4", v1: "[cef][device_custom_string_4][value]"]) } == "silly"
802
+ insist { e.get(ecs_select[disabled: "deviceCustomString4Label", v1: "[cef][device_custom_string_4][label]"]) } == "theory"
803
+ end
654
804
  end
655
- end
656
805
 
657
- context 'with UTF-8 message' do
658
- let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted' }
659
-
660
- # since this spec is encoded UTF-8, the literal strings it contains are encoded with UTF-8,
661
- # but codecs in Logstash tend to receive their input as BINARY (or: ASCII-8BIT); ensure that
662
- # we can handle either without losing the UTF-8 characters from the higher planes.
663
- %w(
664
- BINARY
665
- UTF-8
666
- ).each do |external_encoding|
667
- context "externally encoded as #{external_encoding}" do
668
- let(:message) { super().force_encoding(external_encoding) }
669
- it 'should keep the higher-plane characters' do
670
- decode_one(subject, message.dup) do |event|
671
- validate(event)
672
- insist { event.get("target") } == "aaaaaああああaaaa"
673
- insist { event.get("target").encoding } == Encoding::UTF_8
806
+ context 'with UTF-8 message' do
807
+ let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted' }
808
+
809
+ # since this spec is encoded UTF-8, the literal strings it contains are encoded with UTF-8,
810
+ # but codecs in Logstash tend to receive their input as BINARY (or: ASCII-8BIT); ensure that
811
+ # we can handle either without losing the UTF-8 characters from the higher planes.
812
+ %w(
813
+ BINARY
814
+ UTF-8
815
+ ).each do |external_encoding|
816
+ context "externally encoded as #{external_encoding}" do
817
+ let(:message) { super().force_encoding(external_encoding) }
818
+ it 'should keep the higher-plane characters' do
819
+ decode_one(subject, message.dup) do |event|
820
+ validate(event)
821
+ insist { event.get("target") } == "aaaaaああああaaaa"
822
+ insist { event.get("target").encoding } == Encoding::UTF_8
823
+ end
674
824
  end
675
825
  end
676
826
  end
677
827
  end
678
- end
679
828
 
680
- context 'non-UTF-8 message' do
681
- let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
682
- it 'should emit message unparsed with _cefparsefailure tag' do
683
- decode_one(subject, message.dup) do |event|
684
- insist { event.get("message").bytes.to_a } == message.bytes.to_a
685
- insist { event.get("tags") } == ['_cefparsefailure']
829
+ context 'non-UTF-8 message' do
830
+ let(:logger_stub) { double('Logger').as_null_object }
831
+ before(:each) do
832
+ allow_any_instance_of(described_class).to receive(:logger).and_return(logger_stub)
833
+ end
834
+ let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
835
+ it 'should emit message unparsed with _cefparsefailure tag' do
836
+ decode_one(subject, message.dup) do |event|
837
+ insist { event.get("message").bytes.to_a } == message.bytes.to_a
838
+ insist { event.get("tags") } == ['_cefparsefailure']
839
+ end
840
+ expect(logger_stub).to have_received(:error).with(/Failed to decode CEF payload/, any_args)
686
841
  end
687
842
  end
688
- end
689
843
 
690
- context "with raw_data_field set" do
691
- subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
844
+ context "with raw_data_field set" do
845
+ subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
692
846
 
693
- it "should return the raw message in field message_raw" do
694
- decode_one(subject, message.dup) do |e|
695
- validate(e)
696
- insist { e.get("message_raw") } == message
847
+ it "should return the raw message in field message_raw" do
848
+ decode_one(subject, message.dup) do |e|
849
+ validate(e)
850
+ insist { e.get("message_raw") } == message
851
+ end
852
+ end
853
+ end
854
+
855
+ context "legacy aliases" do
856
+ let(:cef_line) { "CEF:0|security|threatmanager|1.0|100|target acquired|10|destinationLongitude=-73.614830 destinationLatitude=45.505918 sourceLongitude=45.4628328 sourceLatitude=9.1076927" }
857
+
858
+ it ecs_select[disabled:"creates the fields as provided",v1:"maps to ECS fields"] do
859
+ decode_one(codec, cef_line.dup) do |event|
860
+ # |---- LEGACY: AS-PROVIDED ----| |--------- ECS: MAP TO FIELD ----------|
861
+ expect(event.get(ecs_select[disabled:'destinationLongitude',v1:'[destination][geo][location][lon]'])).to eq('-73.614830')
862
+ expect(event.get(ecs_select[disabled:'destinationLatitude', v1:'[destination][geo][location][lat]'])).to eq('45.505918')
863
+ expect(event.get(ecs_select[disabled:'sourceLongitude', v1:'[source][geo][location][lon]' ])).to eq('45.4628328')
864
+ expect(event.get(ecs_select[disabled:'sourceLatitude', v1:'[source][geo][location][lat]' ])).to eq('9.1076927')
865
+ end
697
866
  end
698
867
  end
699
868
  end
700
869
  end
701
870
 
702
- context "encode and decode" do
871
+ context "encode and decode", :ecs_compatibility_support do
703
872
  subject(:codec) { LogStash::Codecs::CEF.new }
704
873
 
705
874
  let(:results) { [] }
706
875
 
707
- it "should return an equal event if encoded and decoded again" do
708
- codec.on_event{|data, newdata| results << newdata}
709
- codec.vendor = "%{deviceVendor}"
710
- codec.product = "%{deviceProduct}"
711
- codec.version = "%{deviceVersion}"
712
- codec.signature = "%{deviceEventClassId}"
713
- codec.name = "%{name}"
714
- codec.severity = "%{severity}"
715
- codec.fields = [ "foo" ]
716
- event = LogStash::Event.new("deviceVendor" => "vendor", "deviceProduct" => "product", "deviceVersion" => "2.0", "deviceEventClassId" => "signature", "name" => "name", "severity" => "1", "foo" => "bar")
717
- codec.encode(event)
718
- codec.decode(results.first) do |e|
719
- expect(e.get('deviceVendor')).to be == event.get('deviceVendor')
720
- expect(e.get('deviceProduct')).to be == event.get('deviceProduct')
721
- expect(e.get('deviceVersion')).to be == event.get('deviceVersion')
722
- expect(e.get('deviceEventClassId')).to be == event.get('deviceEventClassId')
723
- expect(e.get('name')).to be == event.get('name')
724
- expect(e.get('severity')).to be == event.get('severity')
725
- expect(e.get('foo')).to be == event.get('foo')
876
+ ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
877
+ before(:each) do
878
+ allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
879
+ end
880
+
881
+ let(:vendor_field) { ecs_select[disabled:'deviceVendor', v1:'[observer][vendor]'] }
882
+ let(:product_field) { ecs_select[disabled:'deviceProduct', v1:'[observer][product]']}
883
+ let(:version_field) { ecs_select[disabled:'deviceVersion', v1:'[observer][version]']}
884
+ let(:signature_field) { ecs_select[disabled:'deviceEventClassId', v1:'[event][code]']}
885
+ let(:name_field) { ecs_select[disabled:'name', v1:'[cef][name]']}
886
+ let(:severity_field) { ecs_select[disabled:'severity', v1:'[event][severity]']}
887
+
888
+ let(:source_dns_domain_field) { ecs_select[disabled:'sourceDnsDomain',v1:'[source][registered_domain]'] }
889
+
890
+ it "should return an equal event if encoded and decoded again" do
891
+ codec.on_event{|data, newdata| results << newdata}
892
+ codec.vendor = "%{" + vendor_field + "}"
893
+ codec.product = "%{" + product_field + "}"
894
+ codec.version = "%{" + version_field + "}"
895
+ codec.signature = "%{" + signature_field + "}"
896
+ codec.name = "%{" + name_field + "}"
897
+ codec.severity = "%{" + severity_field + "}"
898
+ codec.fields = [ "foo", source_dns_domain_field ]
899
+ event = LogStash::Event.new.tap do |e|
900
+ e.set(vendor_field, "vendor")
901
+ e.set(product_field, "product")
902
+ e.set(version_field, "2.0")
903
+ e.set(signature_field, "signature")
904
+ e.set(name_field, "name")
905
+ e.set(severity_field, "1")
906
+ e.set("foo", "bar")
907
+ e.set(source_dns_domain_field, "apple")
908
+ end
909
+ codec.encode(event)
910
+ codec.decode(results.first) do |e|
911
+ expect(e.get(vendor_field)).to be == event.get(vendor_field)
912
+ expect(e.get(product_field)).to be == event.get(product_field)
913
+ expect(e.get(version_field)).to be == event.get(version_field)
914
+ expect(e.get(signature_field)).to be == event.get(signature_field)
915
+ expect(e.get(name_field)).to be == event.get(name_field)
916
+ expect(e.get(severity_field)).to be == event.get(severity_field)
917
+ expect(e.get('foo')).to be == event.get('foo')
918
+ expect(e.get(source_dns_domain_field)).to be == event.get(source_dns_domain_field)
919
+ end
726
920
  end
727
921
  end
728
922
  end