logstash-codec-cef 6.0.1-java → 6.2.1-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/LICENSE +199 -10
- data/README.md +1 -1
- data/docs/index.asciidoc +368 -9
- data/lib/logstash/codecs/cef.rb +342 -151
- data/lib/logstash/codecs/cef/timestamp_normalizer.rb +115 -0
- data/logstash-codec-cef.gemspec +3 -1
- data/spec/codecs/cef/timestamp_normalizer_spec.rb +274 -0
- data/spec/codecs/cef_spec.rb +480 -308
- metadata +33 -2
@@ -0,0 +1,115 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'java'
|
4
|
+
|
5
|
+
# The CEF specification allows a variety of timestamp formats, some of which
|
6
|
+
# cannot be unambiguously parsed to a specific points in time, and may require
|
7
|
+
# additional side-channel information to do so, namely:
|
8
|
+
# - the time zone or UTC offset (which MAY be included in a separate field)
|
9
|
+
# - the locale (for parsing abbreviated month names)
|
10
|
+
# - the year (assume "recent")
|
11
|
+
#
|
12
|
+
# This normalizer attempts to use the provided context and make reasonable
|
13
|
+
# assumptions when parsing ambiguous dates.
|
14
|
+
class LogStash::Codecs::CEF::TimestampNormalizer
|
15
|
+
|
16
|
+
java_import java.time.Clock
|
17
|
+
java_import java.time.LocalDate
|
18
|
+
java_import java.time.LocalTime
|
19
|
+
java_import java.time.MonthDay
|
20
|
+
java_import java.time.OffsetDateTime
|
21
|
+
java_import java.time.ZoneId
|
22
|
+
java_import java.time.ZonedDateTime
|
23
|
+
java_import java.time.format.DateTimeFormatter
|
24
|
+
java_import java.util.Locale
|
25
|
+
|
26
|
+
def initialize(locale:nil, timezone:nil, clock: Clock.systemUTC)
|
27
|
+
@clock = clock
|
28
|
+
|
29
|
+
java_locale = locale ? get_locale(locale) : Locale.get_default
|
30
|
+
java_timezone = timezone ? ZoneId.of(timezone) : ZoneId.system_default
|
31
|
+
|
32
|
+
@cef_timestamp_format_parser = DateTimeFormatter
|
33
|
+
.ofPattern("MMM dd[ yyyy] HH:mm:ss[.SSSSSSSSS][.SSSSSS][.SSS][ zzz]")
|
34
|
+
.withZone(java_timezone)
|
35
|
+
.withLocale(java_locale)
|
36
|
+
end
|
37
|
+
|
38
|
+
INTEGER_OR_DECIMAL_PATTERN = /\A[1-9][0-9]*(?:\.[0-9]+)?\z/
|
39
|
+
private_constant :INTEGER_OR_DECIMAL_PATTERN
|
40
|
+
|
41
|
+
# @param value [String,Time,Numeric]
|
42
|
+
# The value to parse. `Time`s are returned without modification, and `Numeric` values
|
43
|
+
# are treated as millis-since-epoch (as are fully-numeric strings).
|
44
|
+
# Strings are parsed unsing any of the supported CEF formats, and when the timestamp
|
45
|
+
# does not encode a year, we assume the year from contextual information like the
|
46
|
+
# current time.
|
47
|
+
# @param device_timezone_name [String,nil] (optional):
|
48
|
+
# If known, the time-zone or UTC offset of the device that encoded the timestamp.
|
49
|
+
# This value is used to determine the offset when no offset is encoded in the timestamp.
|
50
|
+
# If not provided, the system default time zone is used instead.
|
51
|
+
# @return [Time]
|
52
|
+
def normalize(value, device_timezone_name=nil)
|
53
|
+
return value if value.kind_of?(Time)
|
54
|
+
|
55
|
+
case value
|
56
|
+
when Numeric then Time.at(Rational(value, 1000))
|
57
|
+
when INTEGER_OR_DECIMAL_PATTERN then Time.at(Rational(value, 1000))
|
58
|
+
else
|
59
|
+
parse_cef_format_string(value.to_s, device_timezone_name)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def get_locale(spec)
|
66
|
+
if spec.nil?
|
67
|
+
Locale.get_default
|
68
|
+
elsif spec =~ /\A([a-z]{2})_([A-Z]{2})\z/
|
69
|
+
lang, country = Regexp.last_match(1), Regexp.last_match(2)
|
70
|
+
Locale.new(lang, country)
|
71
|
+
else
|
72
|
+
Locale.for_language_tag(spec)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def parse_cef_format_string(value, context_timezone=nil)
|
77
|
+
cef_timestamp_format_parser = @cef_timestamp_format_parser
|
78
|
+
cef_timestamp_format_parser = cef_timestamp_format_parser.with_zone(java.time.ZoneId.of(context_timezone)) unless context_timezone.nil?
|
79
|
+
|
80
|
+
parsed_time = cef_timestamp_format_parser.parse_best(value,
|
81
|
+
->(v){ ZonedDateTime.from(v) },
|
82
|
+
->(v){ OffsetDateTime.from(v) },
|
83
|
+
->(v){ resolve_assuming_year(v) }).to_instant
|
84
|
+
|
85
|
+
# Ruby's `Time::at(sec, microseconds_with_frac)`
|
86
|
+
Time.at(parsed_time.get_epoch_second, Rational(parsed_time.get_nano, 1000))
|
87
|
+
rescue => e
|
88
|
+
$stderr.puts "parse_cef_format_sgring(#{value.inspect}, #{context_timezone.inspect}) #!=> #{e.message}"
|
89
|
+
raise
|
90
|
+
end
|
91
|
+
|
92
|
+
def resolve_assuming_year(parsed_temporal_accessor)
|
93
|
+
parsed_monthday = MonthDay.from(parsed_temporal_accessor)
|
94
|
+
parsed_time = LocalTime.from(parsed_temporal_accessor)
|
95
|
+
parsed_zone = ZoneId.from(parsed_temporal_accessor)
|
96
|
+
|
97
|
+
now = ZonedDateTime.now(@clock.with_zone(parsed_zone))
|
98
|
+
|
99
|
+
parsed_timestamp_with_current_year = ZonedDateTime.of(parsed_monthday.at_year(now.get_year), parsed_time, parsed_zone)
|
100
|
+
|
101
|
+
if (parsed_timestamp_with_current_year > now.plus_days(2))
|
102
|
+
# e.g., on May 12, parsing a date from May 15 or later is plausibly from
|
103
|
+
# the prior calendar year and not actually from the future
|
104
|
+
return ZonedDateTime.of(parsed_monthday.at_year(now.get_year - 1), parsed_time, parsed_zone)
|
105
|
+
elsif now.get_month_value == 12 && (parsed_timestamp_with_current_year.plus_years(1) <= now.plus_days(2))
|
106
|
+
# e.g., on December 31, parsing a date from January 1 could plausibly be
|
107
|
+
# from the very-near future but next calendar year due to out-of-sync
|
108
|
+
# clocks, mismatched timezones, etc.
|
109
|
+
return ZonedDateTime.of(parsed_monthday.at_year(now.get_year + 1), parsed_time, parsed_zone)
|
110
|
+
else
|
111
|
+
# otherwise, assume current calendar year
|
112
|
+
return parsed_timestamp_with_current_year
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
data/logstash-codec-cef.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-cef'
|
4
|
-
s.version = '6.
|
4
|
+
s.version = '6.2.1'
|
5
5
|
s.platform = 'java'
|
6
6
|
s.licenses = ['Apache License (2.0)']
|
7
7
|
s.summary = "Reads the ArcSight Common Event Format (CEF)."
|
@@ -22,6 +22,8 @@ Gem::Specification.new do |s|
|
|
22
22
|
|
23
23
|
# Gem dependencies
|
24
24
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
25
|
+
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
|
25
26
|
|
26
27
|
s.add_development_dependency 'logstash-devutils'
|
28
|
+
s.add_development_dependency 'insist'
|
27
29
|
end
|
@@ -0,0 +1,274 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'logstash/util'
|
4
|
+
require "logstash/devutils/rspec/spec_helper"
|
5
|
+
require "insist"
|
6
|
+
require "logstash/codecs/cef"
|
7
|
+
require 'logstash/codecs/cef/timestamp_normalizer'
|
8
|
+
|
9
|
+
describe LogStash::Codecs::CEF::TimestampNormalizer do
|
10
|
+
|
11
|
+
subject(:timestamp_normalizer) { described_class.new }
|
12
|
+
let(:parsed_result) { timestamp_normalizer.normalize(parsable_string) }
|
13
|
+
|
14
|
+
context "parsing dates with a year specified" do
|
15
|
+
let(:parsable_string) { "Jun 17 2027 17:57:06.456" }
|
16
|
+
it 'parses the year correctly' do
|
17
|
+
expect(parsed_result.year).to eq(2027)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context "unparsable inputs" do
|
22
|
+
let(:parsable_string) { "Last Thursday" }
|
23
|
+
it "raises a StandardError exception that can be caught upstream" do
|
24
|
+
expect { parsed_result }.to raise_error(StandardError, /#{Regexp::escape parsable_string}/)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
context "side-channel time zone indicators" do
|
29
|
+
let(:context_timezone) { 'America/New_York' }
|
30
|
+
let(:parsed_result) { timestamp_normalizer.normalize(parsable_string, context_timezone) }
|
31
|
+
|
32
|
+
context "when parsed input does not include offset information" do
|
33
|
+
let(:parsable_string) { "Jun 17 2027 17:57:06.456" }
|
34
|
+
|
35
|
+
it 'offsets to the context timezone time' do
|
36
|
+
expect(parsed_result).to eq(Time.parse("2027-06-17T21:57:06.456Z"))
|
37
|
+
end
|
38
|
+
end
|
39
|
+
context "when parsed input includes offset information" do
|
40
|
+
let(:parsable_string) { "Jun 17 2027 17:57:06.456 -07:00" }
|
41
|
+
|
42
|
+
it 'uses the parsed offset' do
|
43
|
+
expect(parsed_result).to eq(Time.parse("2027-06-18T00:57:06.456Z"))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
context "when parsed input is a millis-since-epoch timestamp" do
|
47
|
+
let(:parsable_string) { "1616623591694" }
|
48
|
+
|
49
|
+
it "does not offset the time" do
|
50
|
+
expect(parsed_result).to eq(Time.at(Rational(1616623591694,1_000)))
|
51
|
+
expect(parsed_result.nsec).to eq(694_000_000)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
context "when parsed input is a millis-since-epoch timestamp with decimal part and microsecond precision" do
|
55
|
+
let(:parsable_string) { "1616623591694.176" }
|
56
|
+
|
57
|
+
it "does not offset the time" do
|
58
|
+
expect(parsed_result).to eq(Time.at(Rational(1616623591694176,1_000_000)))
|
59
|
+
expect(parsed_result.nsec).to eq(694_176_000)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
context "when parsed input is a millis-since-epoch timestamp with decimal part and nanosecond precision" do
|
63
|
+
let(:parsable_string) { "1616623591694.176789" }
|
64
|
+
|
65
|
+
it "does not offset the time" do
|
66
|
+
expect(parsed_result).to eq(Time.at(Rational(1616623591694176789,1_000_000_000)))
|
67
|
+
expect(parsed_result.nsec).to eq(694_176_789)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context "when locale is specified" do
|
73
|
+
let(:locale_language) { 'de' }
|
74
|
+
let(:locale_country) { 'DE' }
|
75
|
+
let(:locale_spec) { "#{locale_language}_#{locale_country}" }
|
76
|
+
|
77
|
+
# Due to locale-provider loading changes in JDK 9, abbreviations for months
|
78
|
+
# depend on a combination of the JDK version and the `java.locale.providers`
|
79
|
+
# system property.
|
80
|
+
# Instead of hard-coding a localized month name, use this process's locales
|
81
|
+
# to generate one.
|
82
|
+
let(:java_locale) { java.util.Locale.new(locale_language, locale_country) }
|
83
|
+
let(:localized_march_abbreviation) do
|
84
|
+
months = java.text.DateFormatSymbols.new(java_locale).get_short_months
|
85
|
+
months[2] # march
|
86
|
+
end
|
87
|
+
|
88
|
+
subject(:timestamp_normalizer) { described_class.new(locale: locale_spec) }
|
89
|
+
|
90
|
+
let(:parsable_string) { "#{localized_march_abbreviation} 17 2019 17:57:06.456 +01:00" }
|
91
|
+
|
92
|
+
it 'uses the locale to parse the date' do
|
93
|
+
expect(parsed_result).to eq(Time.parse("2019-03-17T17:57:06.456+01:00"))
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
context "parsing dates with sub-second precision" do
|
98
|
+
context "whole second precision" do
|
99
|
+
let(:parsable_string) { "Mar 17 2021 12:34:56 +00:00" }
|
100
|
+
it "is accurate to the second" do
|
101
|
+
expect(parsed_result.nsec).to eq(000_000_000)
|
102
|
+
expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56Z"))
|
103
|
+
end
|
104
|
+
end
|
105
|
+
context "millisecond sub-second precision" do
|
106
|
+
let(:parsable_string) { "Mar 17 2021 12:34:56.987" }
|
107
|
+
let(:format_string) { "%b %d %H:%M:%S.%3N" }
|
108
|
+
it "is accurate to the millisecond" do
|
109
|
+
expect(parsed_result.nsec).to eq(987_000_000)
|
110
|
+
expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987Z"))
|
111
|
+
end
|
112
|
+
end
|
113
|
+
context "microsecond sub-second precision" do
|
114
|
+
let(:parsable_string) { "Mar 17 2021 12:34:56.987654" }
|
115
|
+
let(:format_string) { "%b %d %H:%M:%S.%6N" }
|
116
|
+
it "is accurate to the microsecond" do
|
117
|
+
expect(parsed_result.nsec).to eq(987_654_000)
|
118
|
+
expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987654Z"))
|
119
|
+
end
|
120
|
+
end
|
121
|
+
context "nanosecond sub-second precision" do
|
122
|
+
let(:parsable_string) { "Mar 17 2021 12:34:56.987654321" }
|
123
|
+
let(:format_string) { "%b %d %H:%M:%S.%9N" }
|
124
|
+
it "is accurate to the nanosecond" do
|
125
|
+
expect(parsed_result.nsec).to eq(987_654_321)
|
126
|
+
expect(parsed_result).to eq(Time.parse("2021-03-17T12:34:56.987654321Z"))
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
context "parsing dates with no year specified" do
|
132
|
+
let(:time_of_parse) { fail(NotImplementedError) }
|
133
|
+
let(:format_to_parse) { "%b %d %H:%M:%S.%3N" }
|
134
|
+
let(:offset_days) { fail(NotImplementedError) }
|
135
|
+
let(:time_to_parse) { (time_of_parse + (offset_days * 86400)) }
|
136
|
+
let(:parsable_string) { time_to_parse.strftime(format_to_parse) }
|
137
|
+
|
138
|
+
|
139
|
+
let(:anchored_clock) do
|
140
|
+
instant = java.time.Instant.of_epoch_second(time_of_parse.to_i)
|
141
|
+
zone = java.time.ZoneId.system_default
|
142
|
+
|
143
|
+
java.time.Clock.fixed(instant, zone)
|
144
|
+
end
|
145
|
+
|
146
|
+
subject(:timestamp_normalizer) { described_class.new(clock: anchored_clock) }
|
147
|
+
|
148
|
+
let(:parsed_result) { timestamp_normalizer.normalize(parsable_string) }
|
149
|
+
|
150
|
+
context 'when parsing a date during late December' do
|
151
|
+
let(:time_of_parse) { Time.parse("2020-12-31T23:53:08.123456789Z") }
|
152
|
+
context 'and handling a date string from early january' do
|
153
|
+
let(:time_to_parse) { Time.parse("2021-01-01T00:00:08.123456789Z") }
|
154
|
+
it 'assumes that the date being parsed is in the very near future' do
|
155
|
+
expect(parsed_result.month).to eq(1)
|
156
|
+
expect(parsed_result.year).to eq(time_of_parse.year + 1)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
context 'and handling a yearless date string from mid january' do
|
160
|
+
let(:time_to_parse) { Time.parse("2021-01-17T00:00:08.123456789Z") }
|
161
|
+
it 'assumes that the date being parsed is in the distant past' do
|
162
|
+
$stderr.puts(parsable_string)
|
163
|
+
expect(parsed_result.month).to eq(1)
|
164
|
+
expect(parsed_result.year).to eq(time_of_parse.year)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
# As a smoke test to validate the guess-the-year feature when the provided CEF timestamp
|
170
|
+
# does not include the year, we iterate through a variety of dates that we want to parse,
|
171
|
+
# and with each of them we parse with a mock clock as if we were performing the parsing
|
172
|
+
# operation at a variety of date-times relative to the timestamp represented.
|
173
|
+
%w(
|
174
|
+
2021-01-20T04:10:22.961Z
|
175
|
+
2021-06-08T03:38:55.518Z
|
176
|
+
2021-07-12T18:46:12.149Z
|
177
|
+
2021-08-12T04:17:36.680Z
|
178
|
+
2021-08-12T13:20:14.951Z
|
179
|
+
2021-09-17T13:18:57.534Z
|
180
|
+
2021-09-23T16:35:40.404Z
|
181
|
+
2021-10-30T18:52:29.263Z
|
182
|
+
2021-11-11T00:52:39.409Z
|
183
|
+
2021-11-19T13:37:07.189Z
|
184
|
+
2021-12-02T01:09:21.846Z
|
185
|
+
2021-12-11T16:35:05.641Z
|
186
|
+
2021-12-15T14:17:22.152Z
|
187
|
+
2021-12-19T05:53:57.200Z
|
188
|
+
2021-12-20T16:18:17.637Z
|
189
|
+
2021-12-22T12:06:48.965Z
|
190
|
+
2021-12-26T04:45:14.964Z
|
191
|
+
2022-01-05T09:42:39.895Z
|
192
|
+
2022-02-02T04:58:22.080Z
|
193
|
+
2022-02-05T08:10:15.386Z
|
194
|
+
2022-02-15T16:48:27.083Z
|
195
|
+
2022-02-31T13:26:55.298Z
|
196
|
+
2022-03-10T20:16:25.732Z
|
197
|
+
2022-03-20T23:38:58.734Z
|
198
|
+
2022-03-30T03:42:09.546Z
|
199
|
+
2022-04-09T05:55:18.697Z
|
200
|
+
2022-04-14T05:05:29.278Z
|
201
|
+
2022-04-25T15:29:19.567Z
|
202
|
+
2022-05-02T08:34:21.666Z
|
203
|
+
2022-05-24T02:59:02.257Z
|
204
|
+
2022-07-25T01:58:35.713Z
|
205
|
+
2022-07-27T03:27:57.568Z
|
206
|
+
2022-07-28T20:28:22.704Z
|
207
|
+
2022-09-21T08:59:10.508Z
|
208
|
+
2022-10-29T23:54:02.372Z
|
209
|
+
2022-11-12T15:22:51.758Z
|
210
|
+
2022-11-22T22:02:33.278Z
|
211
|
+
2022-12-30T03:18:38.333Z
|
212
|
+
2023-01-02T16:55:57.829Z
|
213
|
+
2023-01-13T16:37:38.078Z
|
214
|
+
2023-01-27T07:27:09.296Z
|
215
|
+
2023-01-30T17:56:43.665Z
|
216
|
+
2023-02-18T11:41:18.886Z
|
217
|
+
2023-02-28T18:51:59.504Z
|
218
|
+
2023-03-10T06:52:14.285Z
|
219
|
+
2023-04-17T16:25:06.489Z
|
220
|
+
2023-04-18T20:46:29.611Z
|
221
|
+
2023-04-27T10:21:41.036Z
|
222
|
+
2023-05-08T02:54:57.131Z
|
223
|
+
2023-05-13T01:17:37.396Z
|
224
|
+
2023-05-24T18:23:05.136Z
|
225
|
+
2023-06-01T11:09:48.129Z
|
226
|
+
2023-06-22T07:44:56.876Z
|
227
|
+
2023-06-25T20:17:44.394Z
|
228
|
+
2023-06-25T20:53:36.329Z
|
229
|
+
2023-07-24T13:07:58.536Z
|
230
|
+
2023-07-27T21:35:54.299Z
|
231
|
+
2023-08-07T11:15:33.803Z
|
232
|
+
2023-08-12T18:45:46.791Z
|
233
|
+
2023-08-19T23:22:19.717Z
|
234
|
+
2023-08-22T23:19:41.075Z
|
235
|
+
2023-08-25T15:22:47.405Z
|
236
|
+
2023-09-03T14:34:13.345Z
|
237
|
+
2023-09-28T05:48:20.040Z
|
238
|
+
2023-09-29T21:14:15.531Z
|
239
|
+
2023-11-12T21:25:55.233Z
|
240
|
+
2023-11-30T00:41:21.834Z
|
241
|
+
2023-12-11T10:14:51.676Z
|
242
|
+
2023-12-14T18:02:33.005Z
|
243
|
+
2023-12-18T09:00:43.589Z
|
244
|
+
2023-12-20T20:02:42.205Z
|
245
|
+
2023-12-22T10:13:37.553Z
|
246
|
+
2023-12-27T19:42:37.905Z
|
247
|
+
2023-12-31T17:52:50.101Z
|
248
|
+
2024-02-29T01:23:45.678Z
|
249
|
+
).map {|ts| Time.parse(ts) }.each do |timestamp|
|
250
|
+
cef_parsable_timestamp = timestamp.strftime("%b %d %H:%M:%S.%3N Z")
|
251
|
+
|
252
|
+
context "when parsing the string `#{cef_parsable_timestamp}`" do
|
253
|
+
|
254
|
+
let(:expected_result) { timestamp }
|
255
|
+
let(:parsable_string) { cef_parsable_timestamp }
|
256
|
+
|
257
|
+
{
|
258
|
+
'very recent past' => -30.789, # ~ 30 seconds ago
|
259
|
+
'somewhat recent past' => -608976.678, # ~ 1 week days ago
|
260
|
+
'distant past' => -29879991.916, # ~ 11-1/2 months days ago
|
261
|
+
'near future' => 132295.719, # ~ 1.5 days from now
|
262
|
+
}.each do |desc, shift|
|
263
|
+
shifted_now = timestamp - shift
|
264
|
+
context "when that string could plausibly be in the #{desc} (NOW: #{shifted_now.iso8601(3)})" do
|
265
|
+
let(:time_of_parse) { shifted_now }
|
266
|
+
it "produces a time in the #{desc} (#{timestamp.iso8601(3)})" do
|
267
|
+
expect(parsed_result).to eq(expected_result)
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
end
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
data/spec/codecs/cef_spec.rb
CHANGED
@@ -1,15 +1,19 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
+
require 'logstash/util'
|
2
3
|
require "logstash/devutils/rspec/spec_helper"
|
4
|
+
require "insist"
|
3
5
|
require "logstash/codecs/cef"
|
4
6
|
require "logstash/event"
|
5
7
|
require "json"
|
6
8
|
|
9
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
|
10
|
+
|
7
11
|
describe LogStash::Codecs::CEF do
|
8
|
-
subject do
|
12
|
+
subject(:codec) do
|
9
13
|
next LogStash::Codecs::CEF.new
|
10
14
|
end
|
11
15
|
|
12
|
-
context "#encode" do
|
16
|
+
context "#encode", :ecs_compatibility_support do
|
13
17
|
subject(:codec) { LogStash::Codecs::CEF.new }
|
14
18
|
|
15
19
|
let(:results) { [] }
|
@@ -209,25 +213,92 @@ describe LogStash::Codecs::CEF do
|
|
209
213
|
codec.encode(event)
|
210
214
|
expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|foo=[0-9TZ.:-]+$/m)
|
211
215
|
end
|
212
|
-
|
213
|
-
it "should encode the CEF field names to their long versions" do
|
214
|
-
# This is with the default value of "reverse_mapping" that is "false".
|
215
|
-
codec.on_event{|data, newdata| results << newdata}
|
216
|
-
codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHost", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
|
217
|
-
event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHost" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
|
218
|
-
codec.encode(event)
|
219
|
-
expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=foobar applicationProtocol=foobar deviceCustomIPv6Address1=foobar deviceCustomIPv6Address1Label=foobar deviceCustomIPv6Address2=foobar deviceCustomIPv6Address2Label=foobar deviceCustomIPv6Address3=foobar deviceCustomIPv6Address3Label=foobar deviceCustomIPv6Address4=foobar deviceCustomIPv6Address4Label=foobar deviceEventCategory=foobar deviceCustomFloatingPoint1=foobar deviceCustomFloatingPoint1Label=foobar deviceCustomFloatingPoint2=foobar deviceCustomFloatingPoint2Label=foobar deviceCustomFloatingPoint3=foobar deviceCustomFloatingPoint3Label=foobar deviceCustomFloatingPoint4=foobar deviceCustomFloatingPoint4Label=foobar deviceCustomNumber1=foobar deviceCustomNumber1Label=foobar deviceCustomNumber2=foobar deviceCustomNumber2Label=foobar deviceCustomNumber3=foobar deviceCustomNumber3Label=foobar baseEventCount=foobar deviceCustomString1=foobar deviceCustomString1Label=foobar deviceCustomString2=foobar deviceCustomString2Label=foobar deviceCustomString3=foobar deviceCustomString3Label=foobar deviceCustomString4=foobar deviceCustomString4Label=foobar deviceCustomString5=foobar deviceCustomString5Label=foobar deviceCustomString6=foobar deviceCustomString6Label=foobar destinationHostName=foobar destinationMacAddress=foobar destinationNtDomain=foobar destinationProcessId=foobar destinationUserPrivileges=foobar destinationProcessName=foobar destinationPort=foobar destinationAddress=foobar destinationUserId=foobar destinationUserName=foobar deviceAddress=foobar deviceHostName=foobar deviceProcessId=foobar endTime=foobar fileName=foobar fileSize=foobar bytesIn=foobar message=foobar bytesOut=foobar eventOutcome=foobar transportProtocol=foobar requestUrl=foobar deviceReceiptTime=foobar sourceHostName=foobar sourceMacAddress=foobar sourceNtDomain=foobar sourceProcessId=foobar sourceUserPrivileges=foobar sourceProcessName=foobar sourcePort=foobar sourceAddress=foobar startTime=foobar sourceUserId=foobar sourceUserName=foobar agentHost=foobar agentReceiptTime=foobar agentType=foobar agentId=foobar agentAddress=foobar agentVersion=foobar agentTimeZone=foobar destinationTimeZone=foobar sourceLongitude=foobar sourceLatitude=foobar destinationLongitude=foobar destinationLatitude=foobar categoryDeviceType=foobar managerReceiptTime=foobar agentMacAddress=foobar$/m)
|
220
|
-
end
|
221
216
|
|
222
|
-
|
223
|
-
|
217
|
+
ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
|
218
|
+
before(:each) do
|
219
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
220
|
+
end
|
224
221
|
|
225
|
-
it "should encode the CEF field names to their
|
222
|
+
it "should encode the CEF field names to their long versions" do
|
223
|
+
# This is with the default value of "reverse_mapping" that is "false".
|
226
224
|
codec.on_event{|data, newdata| results << newdata}
|
227
|
-
codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "
|
228
|
-
event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "
|
225
|
+
codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
|
226
|
+
event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
|
229
227
|
codec.encode(event)
|
230
|
-
expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|
|
228
|
+
expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=foobar applicationProtocol=foobar deviceCustomIPv6Address1=foobar deviceCustomIPv6Address1Label=foobar deviceCustomIPv6Address2=foobar deviceCustomIPv6Address2Label=foobar deviceCustomIPv6Address3=foobar deviceCustomIPv6Address3Label=foobar deviceCustomIPv6Address4=foobar deviceCustomIPv6Address4Label=foobar deviceEventCategory=foobar deviceCustomFloatingPoint1=foobar deviceCustomFloatingPoint1Label=foobar deviceCustomFloatingPoint2=foobar deviceCustomFloatingPoint2Label=foobar deviceCustomFloatingPoint3=foobar deviceCustomFloatingPoint3Label=foobar deviceCustomFloatingPoint4=foobar deviceCustomFloatingPoint4Label=foobar deviceCustomNumber1=foobar deviceCustomNumber1Label=foobar deviceCustomNumber2=foobar deviceCustomNumber2Label=foobar deviceCustomNumber3=foobar deviceCustomNumber3Label=foobar baseEventCount=foobar deviceCustomString1=foobar deviceCustomString1Label=foobar deviceCustomString2=foobar deviceCustomString2Label=foobar deviceCustomString3=foobar deviceCustomString3Label=foobar deviceCustomString4=foobar deviceCustomString4Label=foobar deviceCustomString5=foobar deviceCustomString5Label=foobar deviceCustomString6=foobar deviceCustomString6Label=foobar destinationHostName=foobar destinationMacAddress=foobar destinationNtDomain=foobar destinationProcessId=foobar destinationUserPrivileges=foobar destinationProcessName=foobar destinationPort=foobar destinationAddress=foobar destinationUserId=foobar destinationUserName=foobar deviceAddress=foobar deviceHostName=foobar deviceProcessId=foobar endTime=foobar fileName=foobar fileSize=foobar bytesIn=foobar message=foobar bytesOut=foobar eventOutcome=foobar transportProtocol=foobar requestUrl=foobar deviceReceiptTime=foobar sourceHostName=foobar sourceMacAddress=foobar sourceNtDomain=foobar sourceProcessId=foobar sourceUserPrivileges=foobar sourceProcessName=foobar sourcePort=foobar sourceAddress=foobar startTime=foobar sourceUserId=foobar sourceUserName=foobar agentHostName=foobar agentReceiptTime=foobar agentType=foobar agentId=foobar agentAddress=foobar agentVersion=foobar agentTimeZone=foobar destinationTimeZone=foobar sourceLongitude=foobar sourceLatitude=foobar destinationLongitude=foobar destinationLatitude=foobar categoryDeviceType=foobar managerReceiptTime=foobar agentMacAddress=foobar$/m)
|
229
|
+
end
|
230
|
+
|
231
|
+
if ecs_select.active_mode != :disabled
|
232
|
+
let(:event_flat_hash) do
|
233
|
+
{
|
234
|
+
"[event][action]" => "floop", # deviceAction
|
235
|
+
"[network][protocol]" => "https", # applicationProtocol
|
236
|
+
"[cef][device_custom_ipv6_address_1][value]" => "4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9", # deviceCustomIPv6Address1
|
237
|
+
"[cef][device_custom_ipv6_address_1][label]" => "internal-interface", # deviceCustomIPv6Address1Label
|
238
|
+
"[observer][ip]" => "123.45.67.89", # deviceAddress
|
239
|
+
"[observer][hostname]" => "banana", # deviceHostName
|
240
|
+
"[user_agent][original]" => "'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='", # requestClientApplication
|
241
|
+
"[source][registered_domain]" => "monkey.see" # sourceDnsDomain
|
242
|
+
}
|
243
|
+
end
|
244
|
+
|
245
|
+
let(:event) do
|
246
|
+
event_flat_hash.each_with_object(LogStash::Event.new) do |(fr,v),memo|
|
247
|
+
memo.set(fr, v)
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
it 'encodes the ECS field names to their CEF name' do
|
252
|
+
codec.on_event{|data, newdata| results << newdata}
|
253
|
+
codec.fields = event_flat_hash.keys
|
254
|
+
|
255
|
+
codec.encode(event)
|
256
|
+
|
257
|
+
expect(results.first).to match(%r{^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|deviceAction=floop applicationProtocol=https deviceCustomIPv6Address1=4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9 deviceCustomIPv6Address1Label=internal-interface deviceAddress=123\.45\.67\.89 deviceHostName=banana requestClientApplication='Foo-Bar/2018\.1\.7; Email:user@example\.com; Guid:test\\=' sourceDnsDomain=monkey.see$}m)
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
context "with reverse_mapping set to true" do
|
262
|
+
subject(:codec) { LogStash::Codecs::CEF.new("reverse_mapping" => true) }
|
263
|
+
|
264
|
+
it "should encode the CEF field names to their short versions" do
|
265
|
+
codec.on_event{|data, newdata| results << newdata}
|
266
|
+
codec.fields = [ "deviceAction", "applicationProtocol", "deviceCustomIPv6Address1", "deviceCustomIPv6Address1Label", "deviceCustomIPv6Address2", "deviceCustomIPv6Address2Label", "deviceCustomIPv6Address3", "deviceCustomIPv6Address3Label", "deviceCustomIPv6Address4", "deviceCustomIPv6Address4Label", "deviceEventCategory", "deviceCustomFloatingPoint1", "deviceCustomFloatingPoint1Label", "deviceCustomFloatingPoint2", "deviceCustomFloatingPoint2Label", "deviceCustomFloatingPoint3", "deviceCustomFloatingPoint3Label", "deviceCustomFloatingPoint4", "deviceCustomFloatingPoint4Label", "deviceCustomNumber1", "deviceCustomNumber1Label", "deviceCustomNumber2", "deviceCustomNumber2Label", "deviceCustomNumber3", "deviceCustomNumber3Label", "baseEventCount", "deviceCustomString1", "deviceCustomString1Label", "deviceCustomString2", "deviceCustomString2Label", "deviceCustomString3", "deviceCustomString3Label", "deviceCustomString4", "deviceCustomString4Label", "deviceCustomString5", "deviceCustomString5Label", "deviceCustomString6", "deviceCustomString6Label", "destinationHostName", "destinationMacAddress", "destinationNtDomain", "destinationProcessId", "destinationUserPrivileges", "destinationProcessName", "destinationPort", "destinationAddress", "destinationUserId", "destinationUserName", "deviceAddress", "deviceHostName", "deviceProcessId", "endTime", "fileName", "fileSize", "bytesIn", "message", "bytesOut", "eventOutcome", "transportProtocol", "requestUrl", "deviceReceiptTime", "sourceHostName", "sourceMacAddress", "sourceNtDomain", "sourceProcessId", "sourceUserPrivileges", "sourceProcessName", "sourcePort", "sourceAddress", "startTime", "sourceUserId", "sourceUserName", "agentHostName", "agentReceiptTime", "agentType", "agentId", "agentAddress", "agentVersion", "agentTimeZone", "destinationTimeZone", "sourceLongitude", "sourceLatitude", "destinationLongitude", "destinationLatitude", "categoryDeviceType", "managerReceiptTime", "agentMacAddress" ]
|
267
|
+
event = LogStash::Event.new("deviceAction" => "foobar", "applicationProtocol" => "foobar", "deviceCustomIPv6Address1" => "foobar", "deviceCustomIPv6Address1Label" => "foobar", "deviceCustomIPv6Address2" => "foobar", "deviceCustomIPv6Address2Label" => "foobar", "deviceCustomIPv6Address3" => "foobar", "deviceCustomIPv6Address3Label" => "foobar", "deviceCustomIPv6Address4" => "foobar", "deviceCustomIPv6Address4Label" => "foobar", "deviceEventCategory" => "foobar", "deviceCustomFloatingPoint1" => "foobar", "deviceCustomFloatingPoint1Label" => "foobar", "deviceCustomFloatingPoint2" => "foobar", "deviceCustomFloatingPoint2Label" => "foobar", "deviceCustomFloatingPoint3" => "foobar", "deviceCustomFloatingPoint3Label" => "foobar", "deviceCustomFloatingPoint4" => "foobar", "deviceCustomFloatingPoint4Label" => "foobar", "deviceCustomNumber1" => "foobar", "deviceCustomNumber1Label" => "foobar", "deviceCustomNumber2" => "foobar", "deviceCustomNumber2Label" => "foobar", "deviceCustomNumber3" => "foobar", "deviceCustomNumber3Label" => "foobar", "baseEventCount" => "foobar", "deviceCustomString1" => "foobar", "deviceCustomString1Label" => "foobar", "deviceCustomString2" => "foobar", "deviceCustomString2Label" => "foobar", "deviceCustomString3" => "foobar", "deviceCustomString3Label" => "foobar", "deviceCustomString4" => "foobar", "deviceCustomString4Label" => "foobar", "deviceCustomString5" => "foobar", "deviceCustomString5Label" => "foobar", "deviceCustomString6" => "foobar", "deviceCustomString6Label" => "foobar", "destinationHostName" => "foobar", "destinationMacAddress" => "foobar", "destinationNtDomain" => "foobar", "destinationProcessId" => "foobar", "destinationUserPrivileges" => "foobar", "destinationProcessName" => "foobar", "destinationPort" => "foobar", "destinationAddress" => "foobar", "destinationUserId" => "foobar", "destinationUserName" => "foobar", "deviceAddress" => "foobar", "deviceHostName" => "foobar", "deviceProcessId" => "foobar", "endTime" => "foobar", "fileName" => "foobar", "fileSize" => "foobar", "bytesIn" => "foobar", "message" => "foobar", "bytesOut" => "foobar", "eventOutcome" => "foobar", "transportProtocol" => "foobar", "requestUrl" => "foobar", "deviceReceiptTime" => "foobar", "sourceHostName" => "foobar", "sourceMacAddress" => "foobar", "sourceNtDomain" => "foobar", "sourceProcessId" => "foobar", "sourceUserPrivileges" => "foobar", "sourceProcessName"=> "foobar", "sourcePort" => "foobar", "sourceAddress" => "foobar", "startTime" => "foobar", "sourceUserId" => "foobar", "sourceUserName" => "foobar", "agentHostName" => "foobar", "agentReceiptTime" => "foobar", "agentType" => "foobar", "agentId" => "foobar", "agentAddress" => "foobar", "agentVersion" => "foobar", "agentTimeZone" => "foobar", "destinationTimeZone" => "foobar", "sourceLongitude" => "foobar", "sourceLatitude" => "foobar", "destinationLongitude" => "foobar", "destinationLatitude" => "foobar", "categoryDeviceType" => "foobar", "managerReceiptTime" => "foobar", "agentMacAddress" => "foobar")
|
268
|
+
codec.encode(event)
|
269
|
+
expect(results.first).to match(/^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=foobar app=foobar c6a1=foobar c6a1Label=foobar c6a2=foobar c6a2Label=foobar c6a3=foobar c6a3Label=foobar c6a4=foobar c6a4Label=foobar cat=foobar cfp1=foobar cfp1Label=foobar cfp2=foobar cfp2Label=foobar cfp3=foobar cfp3Label=foobar cfp4=foobar cfp4Label=foobar cn1=foobar cn1Label=foobar cn2=foobar cn2Label=foobar cn3=foobar cn3Label=foobar cnt=foobar cs1=foobar cs1Label=foobar cs2=foobar cs2Label=foobar cs3=foobar cs3Label=foobar cs4=foobar cs4Label=foobar cs5=foobar cs5Label=foobar cs6=foobar cs6Label=foobar dhost=foobar dmac=foobar dntdom=foobar dpid=foobar dpriv=foobar dproc=foobar dpt=foobar dst=foobar duid=foobar duser=foobar dvc=foobar dvchost=foobar dvcpid=foobar end=foobar fname=foobar fsize=foobar in=foobar msg=foobar out=foobar outcome=foobar proto=foobar request=foobar rt=foobar shost=foobar smac=foobar sntdom=foobar spid=foobar spriv=foobar sproc=foobar spt=foobar src=foobar start=foobar suid=foobar suser=foobar ahost=foobar art=foobar at=foobar aid=foobar agt=foobar av=foobar atz=foobar dtz=foobar slong=foobar slat=foobar dlong=foobar dlat=foobar catdt=foobar mrt=foobar amac=foobar$/m)
|
270
|
+
end
|
271
|
+
|
272
|
+
if ecs_select.active_mode != :disabled
|
273
|
+
let(:event_flat_hash) do
|
274
|
+
{
|
275
|
+
"[event][action]" => "floop", # act
|
276
|
+
"[network][protocol]" => "https", # app
|
277
|
+
"[cef][device_custom_ipv6_address_1][value]" => "4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9", # c6a1
|
278
|
+
"[cef][device_custom_ipv6_address_1][label]" => "internal-interface", # c6a1Label
|
279
|
+
"[observer][ip]" => "123.45.67.89", # dvc
|
280
|
+
"[observer][hostname]" => "banana", # dvchost
|
281
|
+
"[user_agent][original]" => "'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='",
|
282
|
+
"[source][registered_domain]" => "monkey.see" # sourceDnsDomain
|
283
|
+
}
|
284
|
+
end
|
285
|
+
|
286
|
+
let(:event) do
|
287
|
+
event_flat_hash.each_with_object(LogStash::Event.new) do |(fr,v),memo|
|
288
|
+
memo.set(fr, v)
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
|
293
|
+
it 'encodes the ECS field names to their CEF keys' do
|
294
|
+
codec.on_event{|data, newdata| results << newdata}
|
295
|
+
codec.fields = event_flat_hash.keys
|
296
|
+
|
297
|
+
codec.encode(event)
|
298
|
+
|
299
|
+
expect(results.first).to match(%r{^CEF:0\|Elasticsearch\|Logstash\|1.0\|Logstash\|Logstash\|6\|act=floop app=https c6a1=4302:c0a5:0bb9:2dfd:7b4e:97f7:a328:98a9 c6a1Label=internal-interface dvc=123\.45\.67\.89 dvchost=banana requestClientApplication='Foo-Bar/2018\.1\.7; Email:user@example\.com; Guid:test\\=' sourceDnsDomain=monkey.see$}m)
|
300
|
+
end
|
301
|
+
end
|
231
302
|
end
|
232
303
|
end
|
233
304
|
end
|
@@ -305,11 +376,21 @@ describe LogStash::Codecs::CEF do
|
|
305
376
|
end
|
306
377
|
end
|
307
378
|
|
308
|
-
|
309
|
-
let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
|
310
|
-
|
379
|
+
module DecodeHelpers
|
311
380
|
def validate(e)
|
312
381
|
insist { e.is_a?(LogStash::Event) }
|
382
|
+
send("validate_ecs_#{ecs_compatibility}", e)
|
383
|
+
end
|
384
|
+
|
385
|
+
def validate_ecs_v1(e)
|
386
|
+
insist { e.get('[cef][version]') } == "0"
|
387
|
+
insist { e.get('[observer][version]') } == "1.0"
|
388
|
+
insist { e.get('[event][code]') } == "100"
|
389
|
+
insist { e.get('[cef][name]') } == "trojan successfully stopped"
|
390
|
+
insist { e.get('[event][severity]') } == "10"
|
391
|
+
end
|
392
|
+
|
393
|
+
def validate_ecs_disabled(e)
|
313
394
|
insist { e.get('cefVersion') } == "0"
|
314
395
|
insist { e.get('deviceVersion') } == "1.0"
|
315
396
|
insist { e.get('deviceEventClassId') } == "100"
|
@@ -333,7 +414,11 @@ describe LogStash::Codecs::CEF do
|
|
333
414
|
fail("Expected one event, got #{events.size} events: #{events.inspect}") unless events.size == 1
|
334
415
|
event = events.first
|
335
416
|
|
336
|
-
|
417
|
+
if block_given?
|
418
|
+
aggregate_failures('decode one') do
|
419
|
+
yield event
|
420
|
+
end
|
421
|
+
end
|
337
422
|
|
338
423
|
event
|
339
424
|
end
|
@@ -359,369 +444,456 @@ describe LogStash::Codecs::CEF do
|
|
359
444
|
|
360
445
|
events
|
361
446
|
end
|
447
|
+
end
|
362
448
|
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
#
|
369
|
-
# Related: https://github.com/elastic/logstash/issues/1645
|
370
|
-
subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
|
449
|
+
context "#decode", :ecs_compatibility_support do
|
450
|
+
ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
|
451
|
+
before(:each) do
|
452
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
453
|
+
end
|
371
454
|
|
372
|
-
|
373
|
-
do_decode(subject,message) do |e|
|
374
|
-
raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
|
375
|
-
end
|
455
|
+
let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
|
376
456
|
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
457
|
+
include DecodeHelpers
|
458
|
+
|
459
|
+
context "with delimiter set" do
|
460
|
+
# '\r\n' in single quotes to simulate the real input from a config
|
461
|
+
# containing \r\n as 4-character sequence in the config:
|
462
|
+
#
|
463
|
+
# delimiter => "\r\n"
|
464
|
+
#
|
465
|
+
# Related: https://github.com/elastic/logstash/issues/1645
|
466
|
+
subject(:codec) { LogStash::Codecs::CEF.new("delimiter" => '\r\n') }
|
467
|
+
|
468
|
+
it "should parse on the delimiter " do
|
469
|
+
do_decode(subject,message) do |e|
|
470
|
+
raise Exception.new("Should not get here. If we do, it means the decoder emitted an event before the delimiter was seen?")
|
471
|
+
end
|
472
|
+
|
473
|
+
decode_one(subject, "\r\n") do |e|
|
474
|
+
validate(e)
|
475
|
+
insist { e.get(ecs_select[disabled: "deviceVendor", v1:"[observer][vendor]"]) } == "security"
|
476
|
+
insist { e.get(ecs_select[disabled: "deviceProduct", v1:"[observer][product]"]) } == "threatmanager"
|
477
|
+
end
|
381
478
|
end
|
382
479
|
end
|
383
|
-
end
|
384
480
|
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
481
|
+
context 'when a CEF header ends with a pair of properly-escaped backslashes' do
|
482
|
+
let(:backslash) { '\\' }
|
483
|
+
let(:pipe) { '|' }
|
484
|
+
let(:message) { "CEF:0|security|threatmanager|1.0|100|double backslash" +
|
485
|
+
backslash + backslash + # escaped backslash
|
486
|
+
backslash + backslash + # escaped backslash
|
487
|
+
"|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
|
392
488
|
|
393
|
-
|
394
|
-
|
489
|
+
it 'should include the backslashes unescaped' do
|
490
|
+
event = decode_one(subject, message)
|
395
491
|
|
396
|
-
|
397
|
-
|
492
|
+
expect(event.get(ecs_select[disabled:'name', v1:'[cef][name]'])).to eq('double backslash' + backslash + backslash )
|
493
|
+
expect(event.get(ecs_select[disabled:'severity',v1:'[event][severity]'])).to eq('10') # ensure we didn't consume the separator
|
494
|
+
end
|
398
495
|
end
|
399
|
-
end
|
400
496
|
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
497
|
+
it "should parse the cef headers" do
|
498
|
+
decode_one(subject, message) do |e|
|
499
|
+
validate(e)
|
500
|
+
insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == "security"
|
501
|
+
insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == "threatmanager"
|
502
|
+
end
|
406
503
|
end
|
407
|
-
end
|
408
504
|
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
505
|
+
it "should parse the cef body" do
|
506
|
+
decode_one(subject, message) do |e|
|
507
|
+
insist { e.get(ecs_select[disabled:"sourceAddress", v1:"[source][ip]"])} == "10.0.0.192"
|
508
|
+
insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
|
509
|
+
insist { e.get(ecs_select[disabled:"sourcePort", v1:"[source][port]"]) } == "1232"
|
510
|
+
end
|
414
511
|
end
|
415
|
-
end
|
416
512
|
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
513
|
+
let (:missing_headers) { "CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
|
514
|
+
it "should be OK with missing CEF headers (multiple pipes in sequence)" do
|
515
|
+
decode_one(subject, missing_headers) do |e|
|
516
|
+
validate(e)
|
517
|
+
insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == ""
|
518
|
+
insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == ""
|
519
|
+
end
|
423
520
|
end
|
424
|
-
end
|
425
521
|
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
522
|
+
let (:leading_whitespace) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232" }
|
523
|
+
it "should strip leading whitespace from the message" do
|
524
|
+
decode_one(subject, leading_whitespace) do |e|
|
525
|
+
validate(e)
|
526
|
+
end
|
430
527
|
end
|
431
|
-
end
|
432
528
|
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
529
|
+
let (:escaped_pipes) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe' }
|
530
|
+
it "should be OK with escaped pipes in the message" do
|
531
|
+
decode_one(subject, escaped_pipes) do |e|
|
532
|
+
insist { e.get("moo") } == 'this\|has an escaped pipe'
|
533
|
+
end
|
437
534
|
end
|
438
|
-
end
|
439
535
|
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
536
|
+
let (:pipes_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe'}
|
537
|
+
it "should be OK with not escaped pipes in the message" do
|
538
|
+
decode_one(subject, pipes_in_message) do |e|
|
539
|
+
insist { e.get("moo") } == 'this|has an pipe'
|
540
|
+
end
|
444
541
|
end
|
445
|
-
end
|
446
542
|
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
543
|
+
# while we may see these in practice, equals MUST be escaped in the extensions per the spec.
|
544
|
+
let (:equal_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\='}
|
545
|
+
it "should be OK with equal in the message" do
|
546
|
+
decode_one(subject, equal_in_message) do |e|
|
547
|
+
insist { e.get("moo") } == 'this =has = equals='
|
548
|
+
end
|
452
549
|
end
|
453
|
-
end
|
454
550
|
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
551
|
+
context "zoneless deviceReceiptTime(rt) when deviceTimeZone(dtz) is provided" do
|
552
|
+
let(:cef_formatted_timestamp) { 'Jul 19 2017 10:50:21.127' }
|
553
|
+
let(:zone_name) { 'Europe/Moscow' }
|
554
|
+
|
555
|
+
let(:utc_timestamp) { Time.iso8601("2017-07-19T07:50:21.127Z") } # In summer of 2017, Europe/Moscow was UTC+03:00
|
556
|
+
|
557
|
+
let(:destination_time_zoned) { %Q{CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|Very-High| eventId=1 msg=Worm successfully stopped art=1500464384997 deviceSeverity=10 rt=#{cef_formatted_timestamp} src=10.0.0.1 sourceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 10.0.0.0-10.255.255.255 spt=1232 dst=2.1.2.2 destinationZoneURI=/All Zones/ArcSight System/Public Address Space Zones/RIPE NCC/2.0.0.0-2.255.255.255 (RIPE NCC) ahost=connector.rhel72 agt=192.168.231.129 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-51-8A-84 av=7.6.0.8009.0 atz=Europe/Lisbon at=syslog_file dvchost=client1 dtz=#{zone_name} _cefVer=0.1 aid=3UBajWl0BABCABBzZSlmUdw==} }
|
558
|
+
|
559
|
+
if ecs_select.active_mode == :disabled
|
560
|
+
it 'persists deviceReceiptTime and deviceTimeZone verbatim' do
|
561
|
+
decode_one(subject, destination_time_zoned) do |event|
|
562
|
+
expect(event.get('deviceReceiptTime')).to eq("Jul 19 2017 10:50:21.127")
|
563
|
+
expect(event.get('deviceTimeZone')).to eq('Europe/Moscow')
|
564
|
+
end
|
565
|
+
end
|
566
|
+
else
|
567
|
+
it 'sets the @timestamp using the value in `rt` combined with the offset provided by `dtz`' do
|
568
|
+
decode_one(subject, destination_time_zoned) do |event|
|
569
|
+
expected_time = LogStash::Timestamp.new(utc_timestamp)
|
570
|
+
expect(event.get('[@timestamp]').to_s).to eq(expected_time.to_s)
|
571
|
+
expect(event.get('[event][timezone]')).to eq(zone_name)
|
572
|
+
end
|
573
|
+
end
|
574
|
+
end
|
575
|
+
end
|
576
|
+
|
577
|
+
let(:malformed_unescaped_equals_in_extension_value) { %q{CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar} }
|
578
|
+
it 'should split correctly' do
|
579
|
+
decode_one(subject, malformed_unescaped_equals_in_extension_value) do |event|
|
580
|
+
expect(event.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"])).to eq('0')
|
581
|
+
expect(event.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"])).to eq('FooBar')
|
582
|
+
expect(event.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"])).to eq('Web Gateway')
|
583
|
+
expect(event.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"])).to eq('1.2.3.45.67')
|
584
|
+
expect(event.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"])).to eq('200')
|
585
|
+
expect(event.get(ecs_select[disabled:"name", v1:"[cef][name]"])).to eq('Success')
|
586
|
+
expect(event.get(ecs_select[disabled:"severity", v1:"[event][severity]"])).to eq('2')
|
587
|
+
|
588
|
+
# extension key/value pairs
|
589
|
+
if ecs_compatibility == :disabled
|
590
|
+
expect(event.get('deviceReceiptTime')).to eq('Sep 07 2018 14:50:39')
|
591
|
+
else
|
592
|
+
expected_time = LogStash::Timestamp.new(Time.parse('Sep 07 2018 14:50:39')).to_s
|
593
|
+
expect(event.get('[@timestamp]').to_s).to eq(expected_time)
|
594
|
+
end
|
595
|
+
expect(event.get(ecs_select[disabled:'deviceEventCategory', v1:'[cef][category]'])).to eq('Access Log')
|
596
|
+
expect(event.get(ecs_select[disabled:'deviceVersion', v1:'[observer][version]'])).to eq('1.2.3.45.67')
|
597
|
+
expect(event.get(ecs_select[disabled:'destinationAddress', v1:'[destination][ip]'])).to eq('1.1.1.1')
|
598
|
+
expect(event.get(ecs_select[disabled:'destinationHostName', v1:'[destination][domain]'])).to eq('foo.example.com')
|
599
|
+
expect(event.get(ecs_select[disabled:'sourceUserName', v1:'[source][user][name]'])).to eq('redacted')
|
600
|
+
expect(event.get(ecs_select[disabled:'sourceAddress', v1:'[source][ip]'])).to eq('2.2.2.2')
|
601
|
+
expect(event.get(ecs_select[disabled:'requestMethod', v1:'[http][request][method]'])).to eq('POST')
|
602
|
+
expect(event.get(ecs_select[disabled:'requestUrl', v1:'[url][original]'])).to eq(%q{'https://foo.example.com/bar/bingo/1'})
|
603
|
+
# Although the value for `requestClientApplication` contains an illegal unquoted equals sign, the sequence
|
604
|
+
# preceeding the unescaped-equals isn't shaped like a key, so we allow it to be a part of the value.
|
605
|
+
expect(event.get(ecs_select[disabled:'requestClientApplication',v1:'[user_agent][original]'])).to eq(%q{'Foo-Bar/2018.1.7; Email:user@example.com; Guid:test='})
|
606
|
+
expect(event.get(ecs_select[disabled:'deviceCustomString1Label',v1:'[cef][device_custom_string_1][label]'])).to eq('Foo Bar')
|
607
|
+
expect(event.get(ecs_select[disabled:'deviceCustomString1', v1:'[cef][device_custom_string_1][value]'])).to eq('')
|
608
|
+
end
|
481
609
|
end
|
482
|
-
end
|
483
610
|
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
611
|
+
context('escaped-equals and unescaped-spaces in the extension values') do
|
612
|
+
let(:query_string) { 'key1=value1&key2=value3 aa.bc&key3=value4'}
|
613
|
+
let(:escaped_query_string) { query_string.gsub('=','\\=') }
|
614
|
+
let(:cef_message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|go=start now query_string=#{escaped_query_string} final=done" }
|
488
615
|
|
489
|
-
|
490
|
-
|
616
|
+
it 'captures the extension values correctly' do
|
617
|
+
event = decode_one(subject, cef_message)
|
491
618
|
|
492
|
-
|
493
|
-
|
494
|
-
|
619
|
+
expect(event.get('go')).to eq('start now')
|
620
|
+
expect(event.get('query_string')).to eq(query_string)
|
621
|
+
expect(event.get('final')).to eq('done')
|
622
|
+
end
|
495
623
|
end
|
496
|
-
end
|
497
624
|
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
625
|
+
let (:escaped_backslash_in_header) {'CEF:0|secu\\\\rity|threat\\\\manager|1.\\\\0|10\\\\0|tro\\\\jan successfully stopped|\\\\10|'}
|
626
|
+
it "should be OK with escaped backslash in the headers" do
|
627
|
+
decode_one(subject, escaped_backslash_in_header) do |e|
|
628
|
+
insist { e.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"]) } == '0'
|
629
|
+
insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'secu\\rity'
|
630
|
+
insist { e.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"]) } == 'threat\\manager'
|
631
|
+
insist { e.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"]) } == '1.\\0'
|
632
|
+
insist { e.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"]) } == '10\\0'
|
633
|
+
insist { e.get(ecs_select[disabled:"name", v1:"[cef][name]"]) } == 'tro\\jan successfully stopped'
|
634
|
+
insist { e.get(ecs_select[disabled:"severity", v1:"[event][severity]"]) } == '\\10'
|
635
|
+
end
|
508
636
|
end
|
509
|
-
end
|
510
637
|
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
638
|
+
let (:escaped_backslash_in_header_edge_case) {'CEF:0|security\\\\\\||threatmanager\\\\|1.0|100|trojan successfully stopped|10|'}
|
639
|
+
it "should be OK with escaped backslash in the headers (edge case: escaped slash in front of pipe)" do
|
640
|
+
decode_one(subject, escaped_backslash_in_header_edge_case) do |e|
|
641
|
+
validate(e)
|
642
|
+
insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'security\\|'
|
643
|
+
insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == 'threatmanager\\'
|
644
|
+
end
|
517
645
|
end
|
518
|
-
end
|
519
646
|
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
647
|
+
let (:escaped_pipes_in_header) {'CEF:0|secu\\|rity|threatmanager\\||1.\\|0|10\\|0|tro\\|jan successfully stopped|\\|10|'}
|
648
|
+
it "should be OK with escaped pipes in the headers" do
|
649
|
+
decode_one(subject, escaped_pipes_in_header) do |e|
|
650
|
+
insist { e.get(ecs_select[disabled:"cefVersion", v1:"[cef][version]"]) } == '0'
|
651
|
+
insist { e.get(ecs_select[disabled:"deviceVendor", v1:"[observer][vendor]"]) } == 'secu|rity'
|
652
|
+
insist { e.get(ecs_select[disabled:"deviceProduct", v1:"[observer][product]"]) } == 'threatmanager|'
|
653
|
+
insist { e.get(ecs_select[disabled:"deviceVersion", v1:"[observer][version]"]) } == '1.|0'
|
654
|
+
insist { e.get(ecs_select[disabled:"deviceEventClassId",v1:"[event][code]"]) } == '10|0'
|
655
|
+
insist { e.get(ecs_select[disabled:"name", v1:"[cef][name]"]) } == 'tro|jan successfully stopped'
|
656
|
+
insist { e.get(ecs_select[disabled:"severity", v1:"[event][severity]"]) } == '|10'
|
657
|
+
end
|
530
658
|
end
|
531
|
-
end
|
532
659
|
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
660
|
+
let (:backslash_in_message) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this \\has \\ backslashs\\'}
|
661
|
+
it "should be OK with backslashs in the message" do
|
662
|
+
decode_one(subject, backslash_in_message) do |e|
|
663
|
+
insist { e.get("moo") } == 'this \\has \\ backslashs\\'
|
664
|
+
end
|
537
665
|
end
|
538
|
-
end
|
539
666
|
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
667
|
+
let (:equal_in_header) {'CEF:0|security|threatmanager=equal|1.0|100|trojan successfully stopped|10|'}
|
668
|
+
it "should be OK with equal in the headers" do
|
669
|
+
decode_one(subject, equal_in_header) do |e|
|
670
|
+
validate(e)
|
671
|
+
insist { e.get(ecs_select[disabled:"deviceProduct",v1:"[observer][product]"]) } == "threatmanager=equal"
|
672
|
+
end
|
545
673
|
end
|
546
|
-
end
|
547
674
|
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
675
|
+
let (:spaces_in_between_keys) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232'}
|
676
|
+
it "should be OK to have one or more spaces between keys" do
|
677
|
+
decode_one(subject, spaces_in_between_keys) do |e|
|
678
|
+
validate(e)
|
679
|
+
insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
|
680
|
+
insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
|
681
|
+
insist { e.get(ecs_select[disabled:"sourcePort",v1:"[source][port]"]) } == "1232"
|
682
|
+
end
|
555
683
|
end
|
556
|
-
end
|
557
684
|
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
685
|
+
let (:dots_in_keys) {'CEF:0|Vendor|Device|Version|13|my message|5|dvchost=loghost cat=traffic deviceSeverity=notice ad.nn=TEST src=192.168.0.1 destinationPort=53'}
|
686
|
+
it "should be OK with dots in keys" do
|
687
|
+
decode_one(subject, dots_in_keys) do |e|
|
688
|
+
insist { e.get(ecs_select[disabled:"deviceHostName",v1:"[observer][hostname]"]) } == "loghost"
|
689
|
+
insist { e.get("ad.nn") } == 'TEST'
|
690
|
+
insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == '192.168.0.1'
|
691
|
+
insist { e.get(ecs_select[disabled:"destinationPort",v1:"[destination][port]"]) } == '53'
|
692
|
+
end
|
565
693
|
end
|
566
|
-
end
|
567
694
|
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
695
|
+
let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 dproc=InternetExplorer x.x.x.x'}
|
696
|
+
it "should be OK to have one or more spaces in values" do
|
697
|
+
decode_one(subject, allow_spaces_in_values) do |e|
|
698
|
+
validate(e)
|
699
|
+
insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
|
700
|
+
insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
|
701
|
+
insist { e.get(ecs_select[disabled:"sourcePort",v1:"[source][port]"]) } == "1232"
|
702
|
+
insist { e.get(ecs_select[disabled:"destinationProcessName",v1:"[destination][process][name]"]) } == "InternetExplorer x.x.x.x"
|
703
|
+
end
|
576
704
|
end
|
577
|
-
end
|
578
705
|
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
706
|
+
let (:preserve_additional_fields_with_dot_notations) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
|
707
|
+
it "should keep ad.fields" do
|
708
|
+
decode_one(subject, preserve_additional_fields_with_dot_notations) do |e|
|
709
|
+
validate(e)
|
710
|
+
insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
|
711
|
+
insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
|
712
|
+
insist { e.get("[ad.field][0]") } == "field0"
|
713
|
+
insist { e.get("[ad.name][1]") } == "new_name"
|
714
|
+
insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
|
715
|
+
insist { e.get('ad.Error_,Code') } == "3221225578"
|
716
|
+
insist { e.get("additional.dotfieldName") } == "new_value"
|
717
|
+
end
|
590
718
|
end
|
591
|
-
end
|
592
719
|
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
720
|
+
let(:preserve_complex_multiple_dot_notation_in_extension_fields) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name' }
|
721
|
+
it "should keep ad.fields" do
|
722
|
+
decode_one(subject, preserve_complex_multiple_dot_notation_in_extension_fields) do |e|
|
723
|
+
validate(e)
|
724
|
+
insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
|
725
|
+
insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
|
726
|
+
insist { e.get("[ad.field][0]") } == "field0"
|
727
|
+
insist { e.get("[ad.foo.name][1]") } == "new_name"
|
728
|
+
insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
|
729
|
+
insist { e.get('ad.Error_,Code') } == "3221225578"
|
730
|
+
insist { e.get("additional.dotfieldName") } == "new_value"
|
731
|
+
end
|
604
732
|
end
|
605
|
-
end
|
606
733
|
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
734
|
+
let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'}
|
735
|
+
it "should correctly parse random values even with additional fields in message" do
|
736
|
+
decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e|
|
737
|
+
validate(e)
|
738
|
+
insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
|
739
|
+
insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
|
740
|
+
insist { e.get("[ad.field][0]") } == "field0"
|
741
|
+
insist { e.get("[ad.name][1]") } == "new_name"
|
742
|
+
insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0"
|
743
|
+
insist { e.get("ad.Error_,Code") } == "3221225578"
|
744
|
+
insist { e.get("additional.dotfieldName") } == "new_value"
|
745
|
+
insist { e.get(ecs_select[disabled:"deviceCustomString4",v1:"[cef][device_custom_string_4][value]"]) } == "401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34"
|
746
|
+
end
|
619
747
|
end
|
620
|
-
end
|
621
748
|
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
749
|
+
let (:preserve_unmatched_key_mappings) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 new_key_by_device=new_values here'}
|
750
|
+
it "should preserve unmatched key mappings" do
|
751
|
+
decode_one(subject, preserve_unmatched_key_mappings) do |e|
|
752
|
+
validate(e)
|
753
|
+
insist { e.get(ecs_select[disabled:"sourceAddress",v1:"[source][ip]"]) } == "10.0.0.192"
|
754
|
+
insist { e.get(ecs_select[disabled:"destinationAddress",v1:"[destination][ip]"]) } == "12.121.122.82"
|
755
|
+
insist { e.get("new_key_by_device") } == "new_values here"
|
756
|
+
end
|
629
757
|
end
|
630
|
-
end
|
631
758
|
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
759
|
+
let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
|
760
|
+
it "should translate most known abbreviated CEF field names" do
|
761
|
+
decode_one(subject, translate_abbreviated_cef_fields) do |e|
|
762
|
+
validate(e)
|
763
|
+
insist { e.get(ecs_select[disabled:"sourceAddress", v1:"[source][ip]"]) } == "10.0.0.192"
|
764
|
+
insist { e.get(ecs_select[disabled:"destinationAddress", v1:"[destination][ip]"]) } == "12.121.122.82"
|
765
|
+
insist { e.get(ecs_select[disabled:"transportProtocol", v1:"[network][transport]"]) } == "TCP"
|
766
|
+
insist { e.get(ecs_select[disabled:"sourceHostName", v1:"[source][domain]"]) } == "source.host.name"
|
767
|
+
insist { e.get(ecs_select[disabled:"destinationHostName",v1:"[destination][domain]"]) } == "destination.host.name"
|
768
|
+
insist { e.get(ecs_select[disabled:"sourcePort", v1:"[source][port]"]) } == "11024"
|
769
|
+
insist { e.get(ecs_select[disabled:"destinationPort", v1:"[destination][port]"]) } == "9200"
|
770
|
+
insist { e.get(ecs_select[disabled:"eventOutcome", v1:"[event][outcome]"]) } == "Success"
|
771
|
+
insist { e.get(ecs_select[disabled:"agentMacAddress", v1:"[agent][mac]"])} == "00:80:48:1c:24:91"
|
772
|
+
end
|
645
773
|
end
|
646
|
-
end
|
647
774
|
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
775
|
+
let (:syslog) { "Syslogdate Sysloghost CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }
|
776
|
+
it "Should detect headers before CEF starts" do
|
777
|
+
decode_one(subject, syslog) do |e|
|
778
|
+
validate(e)
|
779
|
+
insist { e.get(ecs_select[disabled:'syslog',v1:'[log][syslog][header]']) } == 'Syslogdate Sysloghost'
|
780
|
+
end
|
653
781
|
end
|
654
|
-
end
|
655
782
|
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
|
672
|
-
|
783
|
+
context 'with UTF-8 message' do
|
784
|
+
let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted' }
|
785
|
+
|
786
|
+
# since this spec is encoded UTF-8, the literal strings it contains are encoded with UTF-8,
|
787
|
+
# but codecs in Logstash tend to receive their input as BINARY (or: ASCII-8BIT); ensure that
|
788
|
+
# we can handle either without losing the UTF-8 characters from the higher planes.
|
789
|
+
%w(
|
790
|
+
BINARY
|
791
|
+
UTF-8
|
792
|
+
).each do |external_encoding|
|
793
|
+
context "externally encoded as #{external_encoding}" do
|
794
|
+
let(:message) { super().force_encoding(external_encoding) }
|
795
|
+
it 'should keep the higher-plane characters' do
|
796
|
+
decode_one(subject, message.dup) do |event|
|
797
|
+
validate(event)
|
798
|
+
insist { event.get("target") } == "aaaaaああああaaaa"
|
799
|
+
insist { event.get("target").encoding } == Encoding::UTF_8
|
800
|
+
end
|
673
801
|
end
|
674
802
|
end
|
675
803
|
end
|
676
804
|
end
|
677
|
-
end
|
678
805
|
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
806
|
+
context 'non-UTF-8 message' do
|
807
|
+
let(:logger_stub) { double('Logger').as_null_object }
|
808
|
+
before(:each) do
|
809
|
+
allow_any_instance_of(described_class).to receive(:logger).and_return(logger_stub)
|
810
|
+
end
|
811
|
+
let(:message) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=192.168.1.11 target=aaaaaああああaaaa msg=Description Omitted'.encode('SHIFT_JIS') }
|
812
|
+
it 'should emit message unparsed with _cefparsefailure tag' do
|
813
|
+
decode_one(subject, message.dup) do |event|
|
814
|
+
insist { event.get("message").bytes.to_a } == message.bytes.to_a
|
815
|
+
insist { event.get("tags") } == ['_cefparsefailure']
|
816
|
+
end
|
817
|
+
expect(logger_stub).to have_received(:error).with(/Failed to decode CEF payload/, any_args)
|
685
818
|
end
|
686
819
|
end
|
687
|
-
end
|
688
820
|
|
689
|
-
|
690
|
-
|
821
|
+
context "with raw_data_field set" do
|
822
|
+
subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
|
691
823
|
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
824
|
+
it "should return the raw message in field message_raw" do
|
825
|
+
decode_one(subject, message.dup) do |e|
|
826
|
+
validate(e)
|
827
|
+
insist { e.get("message_raw") } == message
|
828
|
+
end
|
829
|
+
end
|
830
|
+
end
|
831
|
+
|
832
|
+
context "legacy aliases" do
|
833
|
+
let(:cef_line) { "CEF:0|security|threatmanager|1.0|100|target acquired|10|destinationLongitude=-73.614830 destinationLatitude=45.505918 sourceLongitude=45.4628328 sourceLatitude=9.1076927" }
|
834
|
+
|
835
|
+
it ecs_select[disabled:"creates the fields as provided",v1:"maps to ECS fields"] do
|
836
|
+
decode_one(codec, cef_line.dup) do |event|
|
837
|
+
# |---- LEGACY: AS-PROVIDED ----| |--------- ECS: MAP TO FIELD ----------|
|
838
|
+
expect(event.get(ecs_select[disabled:'destinationLongitude',v1:'[destination][geo][location][lon]'])).to eq('-73.614830')
|
839
|
+
expect(event.get(ecs_select[disabled:'destinationLatitude', v1:'[destination][geo][location][lat]'])).to eq('45.505918')
|
840
|
+
expect(event.get(ecs_select[disabled:'sourceLongitude', v1:'[source][geo][location][lon]' ])).to eq('45.4628328')
|
841
|
+
expect(event.get(ecs_select[disabled:'sourceLatitude', v1:'[source][geo][location][lat]' ])).to eq('9.1076927')
|
842
|
+
end
|
696
843
|
end
|
697
844
|
end
|
698
845
|
end
|
699
846
|
end
|
700
847
|
|
701
|
-
context "encode and decode" do
|
848
|
+
context "encode and decode", :ecs_compatibility_support do
|
702
849
|
subject(:codec) { LogStash::Codecs::CEF.new }
|
703
850
|
|
704
851
|
let(:results) { [] }
|
705
852
|
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
710
|
-
|
711
|
-
|
712
|
-
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
853
|
+
ecs_compatibility_matrix(:disabled,:v1) do |ecs_select|
|
854
|
+
before(:each) do
|
855
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
856
|
+
end
|
857
|
+
|
858
|
+
let(:vendor_field) { ecs_select[disabled:'deviceVendor', v1:'[observer][vendor]'] }
|
859
|
+
let(:product_field) { ecs_select[disabled:'deviceProduct', v1:'[observer][product]']}
|
860
|
+
let(:version_field) { ecs_select[disabled:'deviceVersion', v1:'[observer][version]']}
|
861
|
+
let(:signature_field) { ecs_select[disabled:'deviceEventClassId', v1:'[event][code]']}
|
862
|
+
let(:name_field) { ecs_select[disabled:'name', v1:'[cef][name]']}
|
863
|
+
let(:severity_field) { ecs_select[disabled:'severity', v1:'[event][severity]']}
|
864
|
+
|
865
|
+
let(:source_dns_domain_field) { ecs_select[disabled:'sourceDnsDomain',v1:'[source][registered_domain]'] }
|
866
|
+
|
867
|
+
it "should return an equal event if encoded and decoded again" do
|
868
|
+
codec.on_event{|data, newdata| results << newdata}
|
869
|
+
codec.vendor = "%{" + vendor_field + "}"
|
870
|
+
codec.product = "%{" + product_field + "}"
|
871
|
+
codec.version = "%{" + version_field + "}"
|
872
|
+
codec.signature = "%{" + signature_field + "}"
|
873
|
+
codec.name = "%{" + name_field + "}"
|
874
|
+
codec.severity = "%{" + severity_field + "}"
|
875
|
+
codec.fields = [ "foo", source_dns_domain_field ]
|
876
|
+
event = LogStash::Event.new.tap do |e|
|
877
|
+
e.set(vendor_field, "vendor")
|
878
|
+
e.set(product_field, "product")
|
879
|
+
e.set(version_field, "2.0")
|
880
|
+
e.set(signature_field, "signature")
|
881
|
+
e.set(name_field, "name")
|
882
|
+
e.set(severity_field, "1")
|
883
|
+
e.set("foo", "bar")
|
884
|
+
e.set(source_dns_domain_field, "apple")
|
885
|
+
end
|
886
|
+
codec.encode(event)
|
887
|
+
codec.decode(results.first) do |e|
|
888
|
+
expect(e.get(vendor_field)).to be == event.get(vendor_field)
|
889
|
+
expect(e.get(product_field)).to be == event.get(product_field)
|
890
|
+
expect(e.get(version_field)).to be == event.get(version_field)
|
891
|
+
expect(e.get(signature_field)).to be == event.get(signature_field)
|
892
|
+
expect(e.get(name_field)).to be == event.get(name_field)
|
893
|
+
expect(e.get(severity_field)).to be == event.get(severity_field)
|
894
|
+
expect(e.get('foo')).to be == event.get('foo')
|
895
|
+
expect(e.get(source_dns_domain_field)).to be == event.get(source_dns_domain_field)
|
896
|
+
end
|
725
897
|
end
|
726
898
|
end
|
727
899
|
end
|